diff --git a/.all-contributorsrc b/.all-contributorsrc deleted file mode 100644 index 2f2d00bd5..000000000 --- a/.all-contributorsrc +++ /dev/null @@ -1,768 +0,0 @@ -{ - "projectName": "kubekey", - "projectOwner": "kubesphere", - "repoType": "github", - "repoHost": "https://github.com", - "files": [ - "README.md", - "CONTRIBUTORS.md", - "README_zh-CN.md" - ], - "imageSize": 100, - "commit": true, - "commitConvention": "none", - "contributors": [ - { - "login": "pixiake", - "name": "pixiake", - "avatar_url": "https://avatars0.githubusercontent.com/u/22290449?v=4", - "profile": "https://github.com/pixiake", - "contributions": [ - "code", - "doc" - ] - }, - { - "login": "Forest-L", - "name": "Forest", - "avatar_url": "https://avatars2.githubusercontent.com/u/50984129?v=4", - "profile": "https://github.com/Forest-L", - "contributions": [ - "code", - "doc" - ] - }, - { - "login": "rayzhou2017", - "name": "rayzhou2017", - "avatar_url": "https://avatars2.githubusercontent.com/u/28859385?v=4", - "profile": "https://kubesphere.io/", - "contributions": [ - "code", - "doc" - ] - }, - { - "login": "shaowenchen", - "name": "shaowenchen", - "avatar_url": "https://avatars2.githubusercontent.com/u/43693241?v=4", - "profile": "https://www.chenshaowen.com/", - "contributions": [ - "code", - "doc" - ] - }, - { - "login": "LinuxSuRen", - "name": "Zhao Xiaojie", - "avatar_url": "https://avatars1.githubusercontent.com/u/1450685?v=4", - "profile": "http://surenpi.com/", - "contributions": [ - "code", - "doc" - ] - }, - { - "login": "zackzhangkai", - "name": "Zack Zhang", - "avatar_url": "https://avatars1.githubusercontent.com/u/20178386?v=4", - "profile": "https://github.com/zackzhangkai", - "contributions": [ - "code" - ] - }, - { - "login": "akhilerm", - "name": "Akhil Mohan", - "avatar_url": "https://avatars1.githubusercontent.com/u/7610845?v=4", - "profile": "https://akhilerm.com/", - "contributions": [ - "code" - ] - }, - { - "login": "FeynmanZhou", - "name": "pengfei", - "avatar_url": "https://avatars3.githubusercontent.com/u/40452856?v=4", - "profile": "https://github.com/FeynmanZhou", - "contributions": [ - "doc" - ] - }, - { - "login": "min-zh", - "name": "min zhang", - "avatar_url": "https://avatars1.githubusercontent.com/u/35321102?v=4", - "profile": "https://github.com/min-zh", - "contributions": [ - "code", - "doc" - ] - }, - { - "login": "zgldh", - "name": "zgldh", - "avatar_url": "https://avatars1.githubusercontent.com/u/312404?v=4", - "profile": "https://github.com/zgldh", - "contributions": [ - "code" - ] - }, - { - "login": "xrjk", - "name": "xrjk", - "avatar_url": "https://avatars0.githubusercontent.com/u/16330256?v=4", - "profile": "https://github.com/xrjk", - "contributions": [ - "code" - ] - }, - { - "login": "stoneshi-yunify", - "name": "yonghongshi", - "avatar_url": "https://avatars2.githubusercontent.com/u/70880165?v=4", - "profile": "https://github.com/stoneshi-yunify", - "contributions": [ - "code" - ] - }, - { - "login": "shenhonglei", - "name": "Honglei", - "avatar_url": "https://avatars2.githubusercontent.com/u/20896372?v=4", - "profile": "https://github.com/shenhonglei", - "contributions": [ - "doc" - ] - }, - { - "login": "liucy1983", - "name": "liucy1983", - "avatar_url": "https://avatars2.githubusercontent.com/u/2360302?v=4", - "profile": "https://github.com/liucy1983", - "contributions": [ - "code" - ] - }, - { - "login": "lilien1010", - "name": "Lien", - "avatar_url": "https://avatars1.githubusercontent.com/u/3814966?v=4", - "profile": "https://github.com/lilien1010", - "contributions": [ - "doc" - ] - }, - { - "login": "klj890", - "name": "Tony Wang", - "avatar_url": "https://avatars3.githubusercontent.com/u/19380605?v=4", - "profile": "https://github.com/klj890", - "contributions": [ - "doc" - ] - }, - { - "login": "hlwanghl", - "name": "Hongliang Wang", - "avatar_url": "https://avatars3.githubusercontent.com/u/4861515?v=4", - "profile": "https://github.com/hlwanghl", - "contributions": [ - "code" - ] - }, - { - "login": "fafucoder", - "name": "dawn", - "avatar_url": "https://avatars0.githubusercontent.com/u/16442491?v=4", - "profile": "https://fafucoder.github.io/", - "contributions": [ - "code" - ] - }, - { - "login": "duanjiong", - "name": "Duan Jiong", - "avatar_url": "https://avatars1.githubusercontent.com/u/3678855?v=4", - "profile": "https://github.com/duanjiong", - "contributions": [ - "code" - ] - }, - { - "login": "calvinyv", - "name": "calvinyv", - "avatar_url": "https://avatars3.githubusercontent.com/u/28883416?v=4", - "profile": "https://github.com/calvinyv", - "contributions": [ - "doc" - ] - }, - { - "login": "benjaminhuo", - "name": "Benjamin Huo", - "avatar_url": "https://avatars2.githubusercontent.com/u/18525465?v=4", - "profile": "https://github.com/benjaminhuo", - "contributions": [ - "doc" - ] - }, - { - "login": "Sherlock113", - "name": "Sherlock113", - "avatar_url": "https://avatars2.githubusercontent.com/u/65327072?v=4", - "profile": "https://github.com/Sherlock113", - "contributions": [ - "doc" - ] - }, - { - "login": "Fuchange", - "name": "fu_changjie", - "avatar_url": "https://avatars1.githubusercontent.com/u/31716848?v=4", - "profile": "https://github.com/Fuchange", - "contributions": [ - "doc" - ] - }, - { - "login": "yuswift", - "name": "yuswift", - "avatar_url": "https://avatars1.githubusercontent.com/u/37265389?v=4", - "profile": "https://github.com/yuswift", - "contributions": [ - "code" - ] - }, - { - "login": "ruiyaoOps", - "name": "ruiyaoOps", - "avatar_url": "https://avatars.githubusercontent.com/u/35256376?v=4", - "profile": "https://github.com/ruiyaoOps", - "contributions": [ - "doc" - ] - }, - { - "login": "lxm", - "name": "LXM", - "avatar_url": "https://avatars.githubusercontent.com/u/1918195?v=4", - "profile": "http://www.luxingmin.com", - "contributions": [ - "doc" - ] - }, - { - "login": "sbhnet", - "name": "sbhnet", - "avatar_url": "https://avatars.githubusercontent.com/u/2368131?v=4", - "profile": "https://github.com/sbhnet", - "contributions": [ - "code" - ] - }, - { - "login": "misteruly", - "name": "misteruly", - "avatar_url": "https://avatars.githubusercontent.com/u/31399968?v=4", - "profile": "https://github.com/misteruly", - "contributions": [ - "code" - ] - }, - { - "login": "JohnNiang", - "name": "John Niang", - "avatar_url": "https://avatars.githubusercontent.com/u/16865714?v=4", - "profile": "https://johnniang.me", - "contributions": [ - "doc" - ] - }, - { - "login": "alimy", - "name": "Michael Li", - "avatar_url": "https://avatars.githubusercontent.com/u/10525842?v=4", - "profile": "https://alimy.me", - "contributions": [ - "code" - ] - }, - { - "login": "duguhaotian", - "name": "独孤昊天", - "avatar_url": "https://avatars.githubusercontent.com/u/3174621?v=4", - "profile": "https://github.com/duguhaotian", - "contributions": [ - "code" - ] - }, - { - "login": "lshmouse", - "name": "Liu Shaohui", - "avatar_url": "https://avatars.githubusercontent.com/u/118687?v=4", - "profile": "https://github.com/lshmouse", - "contributions": [ - "code" - ] - }, - { - "login": "24sama", - "name": "Leo Li", - "avatar_url": "https://avatars.githubusercontent.com/u/43993589?v=4", - "profile": "https://github.com/24sama", - "contributions": [ - "code" - ] - }, - { - "login": "RolandMa1986", - "name": "Roland", - "avatar_url": "https://avatars.githubusercontent.com/u/1720333?v=4", - "profile": "https://github.com/RolandMa1986", - "contributions": [ - "code" - ] - }, - { - "login": "vinsonzou", - "name": "Vinson Zou", - "avatar_url": "https://avatars.githubusercontent.com/u/2347587?v=4", - "profile": "https://ops.m114.org", - "contributions": [ - "doc" - ] - }, - { - "login": "tagGeeY", - "name": "tag_gee_y", - "avatar_url": "https://avatars.githubusercontent.com/u/35259969?v=4", - "profile": "https://github.com/tagGeeY", - "contributions": [ - "code" - ] - }, - { - "login": "liulangwa", - "name": "codebee", - "avatar_url": "https://avatars.githubusercontent.com/u/25916792?v=4", - "profile": "https://github.com/liulangwa", - "contributions": [ - "code" - ] - }, - { - "login": "TheApeMachine", - "name": "Daniel Owen van Dommelen", - "avatar_url": "https://avatars.githubusercontent.com/u/9572060?v=4", - "profile": "https://github.com/TheApeMachine", - "contributions": [ - "ideas" - ] - }, - { - "login": "Naidile-P-N", - "name": "Naidile P N", - "avatar_url": "https://avatars.githubusercontent.com/u/29476402?v=4", - "profile": "https://github.com/Naidile-P-N", - "contributions": [ - "code" - ] - }, - { - "login": "haiker2011", - "name": "Haiker Sun", - "avatar_url": "https://avatars.githubusercontent.com/u/8073429?v=4", - "profile": "https://github.com/haiker2011", - "contributions": [ - "code" - ] - }, - { - "login": "yj-cloud", - "name": "Jing Yu", - "avatar_url": "https://avatars.githubusercontent.com/u/19648473?v=4", - "profile": "https://github.com/yj-cloud", - "contributions": [ - "code" - ] - }, - { - "login": "chaunceyjiang", - "name": "Chauncey", - "avatar_url": "https://avatars.githubusercontent.com/u/17962021?v=4", - "profile": "https://github.com/chaunceyjiang", - "contributions": [ - "code" - ] - }, - { - "login": "tanguofu", - "name": "Tan Guofu", - "avatar_url": "https://avatars.githubusercontent.com/u/87045830?v=4", - "profile": "https://github.com/tanguofu", - "contributions": [ - "code" - ] - }, - { - "login": "lvillis", - "name": "lvillis", - "avatar_url": "https://avatars.githubusercontent.com/u/56720445?v=4", - "profile": "https://github.com/lvillis", - "contributions": [ - "doc" - ] - }, - { - "login": "vincenthe11", - "name": "Vincent He", - "avatar_url": "https://avatars.githubusercontent.com/u/8400716?v=4", - "profile": "https://github.com/vincenthe11", - "contributions": [ - "code" - ] - }, - { - "login": "tpiperatgod", - "name": "laminar", - "avatar_url": "https://avatars.githubusercontent.com/u/2360535?v=4", - "profile": "https://laminar.fun/", - "contributions": [ - "code" - ] - }, - { - "login": "cumirror", - "name": "tongjin", - "avatar_url": "https://avatars.githubusercontent.com/u/2455429?v=4", - "profile": "https://github.com/cumirror", - "contributions": [ - "code" - ] - }, - { - "login": "muzi502", - "name": "Reimu", - "avatar_url": "https://avatars.githubusercontent.com/u/42566386?v=4", - "profile": "http://k8s.li", - "contributions": [ - "code" - ] - }, - { - "login": "eltociear", - "name": "Ikko Ashimine", - "avatar_url": "https://avatars.githubusercontent.com/u/22633385?v=4", - "profile": "https://bandism.net/", - "contributions": [ - "doc" - ] - }, - { - "login": "yeya24", - "name": "Ben Ye", - "avatar_url": "https://avatars.githubusercontent.com/u/25150124?v=4", - "profile": "https://yeya24.github.io/", - "contributions": [ - "code" - ] - }, - { - "login": "yinheli", - "name": "yinheli", - "avatar_url": "https://avatars.githubusercontent.com/u/235094?v=4", - "profile": "https://github.com/yinheli", - "contributions": [ - "code" - ] - }, - { - "login": "hellocn9", - "name": "hellocn9", - "avatar_url": "https://avatars.githubusercontent.com/u/102210430?v=4", - "profile": "https://github.com/hellocn9", - "contributions": [ - "code" - ] - }, - { - "login": "brandan-schmitz", - "name": "Brandan Schmitz", - "avatar_url": "https://avatars.githubusercontent.com/u/6267549?v=4", - "profile": "https://github.com/brandan-schmitz", - "contributions": [ - "code" - ] - }, - { - "login": "yjqg6666", - "name": "yjqg6666", - "avatar_url": "https://avatars.githubusercontent.com/u/1879641?v=4", - "profile": "https://github.com/yjqg6666", - "contributions": [ - "doc", - "code" - ] - }, - { - "login": "zaunist", - "name": "失眠是真滴难受", - "avatar_url": "https://avatars.githubusercontent.com/u/38528079?v=4", - "profile": "https://github.com/zaunist", - "contributions": [ - "code" - ] - }, - { - "login": "mangoGoForward", - "name": "mango", - "avatar_url": "https://avatars.githubusercontent.com/u/35127166?v=4", - "profile": "https://github.com/mangoGoForward", - "contributions": [ - "review" - ] - }, - { - "login": "wenwutang1", - "name": "wenwutang", - "avatar_url": "https://avatars.githubusercontent.com/u/45817987?v=4", - "profile": "https://github.com/wenwutang1", - "contributions": [ - "code" - ] - }, - { - "login": "kuops", - "name": "Shiny Hou", - "avatar_url": "https://avatars.githubusercontent.com/u/18283256?v=4", - "profile": "http://kuops.com", - "contributions": [ - "code" - ] - }, - { - "login": "zhouqiu0103", - "name": "zhouqiu0103", - "avatar_url": "https://avatars.githubusercontent.com/u/108912268?v=4", - "profile": "https://github.com/zhouqiu0103", - "contributions": [ - "code" - ] - }, - { - "login": "77yu77", - "name": "77yu77", - "avatar_url": "https://avatars.githubusercontent.com/u/73932296?v=4", - "profile": "https://github.com/77yu77", - "contributions": [ - "code" - ] - }, - { - "login": "hzhhong", - "name": "hzhhong", - "avatar_url": "https://avatars.githubusercontent.com/u/83079531?v=4", - "profile": "https://github.com/hzhhong", - "contributions": [ - "code" - ] - }, - { - "login": "arugal", - "name": "zhang-wei", - "avatar_url": "https://avatars.githubusercontent.com/u/26432832?v=4", - "profile": "https://github.com/arugal", - "contributions": [ - "code" - ] - }, - { - "login": "xiaods", - "name": "Deshi Xiao", - "avatar_url": "https://avatars.githubusercontent.com/u/37678?v=4", - "profile": "https://twitter.com/xds2000", - "contributions": [ - "code", - "doc" - ] - }, - { - "login": "besscroft", - "name": "besscroft", - "avatar_url": "https://avatars.githubusercontent.com/u/33775809?v=4", - "profile": "https://besscroft.com", - "contributions": [ - "doc" - ] - }, - { - "login": "zhangzhiqiangcs", - "name": "张志强", - "avatar_url": "https://avatars.githubusercontent.com/u/8319897?v=4", - "profile": "https://github.com/zhangzhiqiangcs", - "contributions": [ - "code" - ] - }, - { - "login": "lwabish", - "name": "lwabish", - "avatar_url": "https://avatars.githubusercontent.com/u/7044019?v=4", - "profile": "https://github.com/lwabish", - "contributions": [ - "code", - "doc" - ] - }, - { - "login": "qyz87", - "name": "qyz87", - "avatar_url": "https://avatars.githubusercontent.com/u/36068894?v=4", - "profile": "https://github.com/qyz87", - "contributions": [ - "code" - ] - }, - { - "login": "fangzhengjin", - "name": "ZhengJin Fang", - "avatar_url": "https://avatars.githubusercontent.com/u/12680972?v=4", - "profile": "https://github.com/fangzhengjin", - "contributions": [ - "code" - ] - }, - { - "login": "ExerciseBook", - "name": "Eric_Lian", - "avatar_url": "https://avatars.githubusercontent.com/u/6327311?v=4", - "profile": "http://lhr.wiki", - "contributions": [ - "code" - ] - }, - { - "login": "nicognaW", - "name": "nicognaw", - "avatar_url": "https://avatars.githubusercontent.com/u/66731869?v=4", - "profile": "https://github.com/nicognaW", - "contributions": [ - "code" - ] - }, - { - "login": "deqingLv", - "name": "吕德庆", - "avatar_url": "https://avatars.githubusercontent.com/u/6064297?v=4", - "profile": "https://github.com/deqingLv", - "contributions": [ - "code" - ] - }, - { - "login": "littleplus", - "name": "littleplus", - "avatar_url": "https://avatars.githubusercontent.com/u/11694750?v=4", - "profile": "https://github.com/littleplus", - "contributions": [ - "code" - ] - }, - { - "login": "Nello-Angelo", - "name": "Konstantin", - "avatar_url": "https://avatars.githubusercontent.com/u/82488489?v=4", - "profile": "https://www.linkedin.com/in/%D0%BA%D0%BE%D0%BD%D1%81%D1%82%D0%B0%D0%BD%D1%82%D0%B8%D0%BD-%D0%B0%D0%BA%D0%B0%D0%BA%D0%B8%D0%B5%D0%B2-13130b1b4/", - "contributions": [ - "ideas" - ] - }, - { - "login": "kiragoo", - "name": "kiragoo", - "avatar_url": "https://avatars.githubusercontent.com/u/7400711?v=4", - "profile": "https://kiragoo.github.io", - "contributions": [ - "code" - ] - }, - { - "login": "jojotong", - "name": "jojotong", - "avatar_url": "https://avatars.githubusercontent.com/u/100849526?v=4", - "profile": "https://github.com/jojotong", - "contributions": [ - "code" - ] - }, - { - "login": "littleBlackHouse", - "name": "littleBlackHouse", - "avatar_url": "https://avatars.githubusercontent.com/u/54946465?v=4", - "profile": "https://github.com/littleBlackHouse", - "contributions": [ - "code", - "doc" - ] - }, - { - "login": "testwill", - "name": "guangwu", - "avatar_url": "https://avatars.githubusercontent.com/u/8717479?v=4", - "profile": "https://github.com/testwill", - "contributions": [ - "code", - "doc" - ] - }, - { - "login": "wongearl", - "name": "wongearl", - "avatar_url": "https://avatars.githubusercontent.com/u/36498442?v=4", - "profile": "https://github.com/wongearl", - "contributions": [ - "code" - ] - }, - { - "login": "wenwenxiong", - "name": "wenwenxiong", - "avatar_url": "https://avatars.githubusercontent.com/u/10548812?v=4", - "profile": "https://github.com/wenwenxiong", - "contributions": [ - "code" - ] - }, - { - "login": "BaiMeow", - "name": "柏喵Sakura", - "avatar_url": "https://avatars.githubusercontent.com/u/38121125?v=4", - "profile": "https://baimeow.cn/", - "contributions": [ - "code" - ] - }, - { - "login": "cuishuang", - "name": "cui fliter", - "avatar_url": "https://avatars.githubusercontent.com/u/15921519?v=4", - "profile": "https://dashen.tech", - "contributions": [ - "doc" - ] - }, - { - "login": "liuxu623", - "name": "刘旭", - "avatar_url": "https://avatars.githubusercontent.com/u/9653438?v=4", - "profile": "https://github.com/liuxu623", - "contributions": [ - "code" - ] - } - ], - "contributorsPerLine": 7, - "skipCi": true, - "commitType": "docs" -} diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..1cf278fd1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,35 @@ +# Binaries for programs and plugins +*.exe +*.dll +*.so +*.dylib +*.tmp +.DS_Store + +# Test binary, build with `go test -c` +*.test + +# IntelliJ +.idea/ +*.iml + +# Vscode files +.vscode + +# rbac and manager config for example provider +manager_image_patch.yaml-e +manager_pull_policy.yaml-e + +# Sample config files auto-generated by kubebuilder +config/samples + +# test results +_artifacts + +# Used during parts of the build process. Files _should_ get cleaned up automatically. +# This is also a good location for any temporary manfiests used during development +tmp + +# Used by current object +/_output/ +dist/ diff --git a/.github/OWNERS b/.github/OWNERS index 45431bbfa..5cc21bae8 100644 --- a/.github/OWNERS +++ b/.github/OWNERS @@ -1,18 +1,17 @@ approvers: - pixiake - - Forest-L - rayzhou2017 - LinuxSuRen + - ImitationImmortal reviewers: - pixiake - - Forest-L - rayzhou2017 - zryfish - - shaowenchen - benjaminhuo - calvinyv - FeynmanZhou - huanggze - wansir - LinuxSuRen + - ImitationImmortal diff --git a/.github/dependabot.yml b/.github/dependabot.yaml similarity index 100% rename from .github/dependabot.yml rename to .github/dependabot.yaml diff --git a/.github/workflows/build-multiarch.yaml b/.github/workflows/build-multiarch.yaml index e0be29add..83a2f1361 100644 --- a/.github/workflows/build-multiarch.yaml +++ b/.github/workflows/build-multiarch.yaml @@ -3,36 +3,48 @@ name: BuildContainerImage on: push: tags: - - 'v3*' + - 'v4*' jobs: build: name: Build runs-on: ubuntu-latest + if: github.repository == 'kubesphere/kubekey' env: GO111MODULE: on steps: - - name: Set env - run: echo "RELEASE_TAG=${GITHUB_REF:10}" >> $GITHUB_ENV + - name: Prepare + id: prepare + run: | + VERSION=latest + + if [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/} + fi + echo "version=${VERSION}" >> "$GITHUB_OUTPUT" + + - name: Checkout code + uses: actions/checkout@v5 - - name: Check out code into the Go module directory - uses: actions/checkout@v3 + - name: Login to DOCKER + uses: docker/login-action@v3 with: - fetch-depth: 0 + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 with: platforms: all - name: Set up Docker buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - - name: Build and push docker images - env: - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - if: github.event_name == 'push' + - name: Build and push Docker images run: | - echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin - make release-prod \ No newline at end of file + tag=${{ steps.prepare.outputs.version }} + if [[ $tag =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + REGISTRY=docker.io/kubesphere TAG=$tag make docker-push + else + REGISTRY=docker.io/kubespheredev TAG=$tag make docker-push + fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 5d480c5fa..000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: CI - -on: - push: - branches: - - 'master' - - 'release*' - tags: - - 'v*' - pull_request: - branches: - - 'master' - -jobs: - build: - name: Build - runs-on: ubuntu-latest - env: - GO111MODULE: on - steps: - - - name: Set up Go 1.19 - uses: actions/setup-go@v3 - with: - go-version: 1.19 - id: go - - - name: Check out code into the Go module directory - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - - name: Downloading go dependencies - run: go mod tidy - - # If there are any diffs from goimports or go mod tidy, fail. - - name: Verify no changes from goimports and go mod tidy. - run: | - if [ -n "$(git status --porcelain)" ]; then - echo 'To fix this check, run "go mod tidy"' - git status # Show the files that failed to pass the check. - exit 1 - fi - - - name: Build command-line tool - run: | - make kk diff --git a/.github/workflows/gen-repository-iso.yaml b/.github/workflows/gen-repository-iso.yaml index 23428a9a2..c26b3fcb7 100644 --- a/.github/workflows/gen-repository-iso.yaml +++ b/.github/workflows/gen-repository-iso.yaml @@ -1,9 +1,9 @@ --- -name: gen-repository-iso +name: GenRepositoryISO on: push: tags: - - 'v*' + - 'ISO-*' workflow_dispatch: jobs: @@ -14,16 +14,14 @@ jobs: fail-fast: false matrix: include: - - name: centos7-rpms - dockerfile: dockerfile.centos7 - name: almalinux-9.0-rpms dockerfile: dockerfile.almalinux90 - name: debian10-debs dockerfile: dockerfile.debian10 - name: debian11-debs dockerfile: dockerfile.debian11 - - name: ubuntu-16.04-debs - dockerfile: dockerfile.ubuntu1604 + - name: debian12-debs + dockerfile: dockerfile.debian12 - name: ubuntu-18.04-debs dockerfile: dockerfile.ubuntu1804 - name: ubuntu-20.04-debs @@ -32,16 +30,16 @@ jobs: dockerfile: dockerfile.ubuntu2204 steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v5 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Build iso image to local - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 with: context: hack/gen-repository-iso file: hack/gen-repository-iso/${{ matrix.dockerfile }} @@ -56,7 +54,7 @@ jobs: sha256sum *.iso > ${{ matrix.name }}.iso.sha256sum.txt - name: Wait for release workflow to finish - uses: lewagon/wait-on-check-action@v1.3.1 + uses: lewagon/wait-on-check-action@v1.3.4 with: ref: ${{ github.ref }} check-name: 'create draft release' @@ -66,7 +64,7 @@ jobs: - name: Release and upload packages if: startsWith(github.ref, 'refs/tags/') - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -74,4 +72,4 @@ jobs: files: | ${{ matrix.name }}.iso.sha256sum.txt ${{ matrix.name }}-amd64.iso - ${{ matrix.name }}-arm64.iso + ${{ matrix.name }}-arm64.iso \ No newline at end of file diff --git a/.github/workflows/golangci-lint.yaml b/.github/workflows/golangci-lint.yaml new file mode 100644 index 000000000..11628d547 --- /dev/null +++ b/.github/workflows/golangci-lint.yaml @@ -0,0 +1,81 @@ +name: GolangCILint + +on: + pull_request: + types: [opened, edited, synchronize, reopened] + paths: + - 'pkg/**' + - 'cmd/**' + - 'version/**' + - 'go.mod' + - 'go.sum' + - 'api/**' + +# Remove all permissions from GITHUB_TOKEN except metadata. +permissions: {} + +jobs: + golangci: + name: lint + runs-on: ubuntu-latest + if: github.repository == 'kubesphere/kubekey' + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Setup golang + uses: actions/setup-go@v5 + with: + go-version: 1.23.3 + + - name: Sync mod + run: make generate-modules + + - name: golangci-lint + uses: golangci/golangci-lint-action@v6 + with: + version: v1.63.3 + + verify: + name: verify + runs-on: ubuntu-latest + if: github.repository == 'kubesphere/kubekey' + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Setup golang + uses: actions/setup-go@v5 + with: + go-version: 1.23.3 + + - name: Sync mod + run: make generate-modules + + - name: Verify + run: ALL_VERIFY_CHECKS="goimports releaser" make verify + + test: + name: test + runs-on: ubuntu-latest + if: github.repository == 'kubesphere/kubekey' + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Setup golang + uses: actions/setup-go@v5 + with: + go-version: 1.23.3 + + - name: Sync mod + run: make generate-modules + + - name: Test + run: make test diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml deleted file mode 100644 index 04b4448ca..000000000 --- a/.github/workflows/golangci-lint.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: golangci-lint - -on: - pull_request: - types: [opened, edited, synchronize, reopened] - -# Remove all permissions from GITHUB_TOKEN except metadata. -permissions: {} - -jobs: - golangci: - name: lint - runs-on: ubuntu-latest - strategy: - matrix: - working-directory: - - "" - - test - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version: 1.19 - - name: golangci-lint - uses: golangci/golangci-lint-action@v3.6.0 - with: - version: v1.50.1 - working-directory: ${{matrix.working-directory}} diff --git a/.github/workflows/kubernetes-auto-support.yaml b/.github/workflows/kubernetes-auto-support.yaml deleted file mode 100644 index 766365fe8..000000000 --- a/.github/workflows/kubernetes-auto-support.yaml +++ /dev/null @@ -1,70 +0,0 @@ ---- -name: Kubernetes-Auto-Support - -on: - schedule: - - cron: '0 0 * * *' - -jobs: - update-kubernetes-version: - runs-on: ubuntu-latest - if: github.repository == 'kubesphere/kubekey' - steps: - - uses: actions/checkout@v3 - - - name: Set up Go 1.19 - uses: actions/setup-go@v3 - with: - go-version: 1.19 - id: go - - - name: install dependiencies - run: | - sudo apt update - sudo apt install skopeo -y - pip install natsort - wget https://attack-on-titan.gd2.qingstor.com/qsctl/v2.4.3/qsctl_v2.4.3_linux_amd64.tar.gz - tar -zxvf qsctl_v2.4.3_linux_amd64.tar.gz - mv qsctl_v2.4.3_linux_amd64 /usr/local/bin/qsctl - rm -rf qsctl_v2.4.3_linux_amd64.tar.gz - - - name: update components.json - id: get_new_version - run: | - chmod +x hack/auto-update-version.py - hack/auto-update-version.py - [ -f version.tmp ] && echo "UPDATE_VERSION=true" >> $GITHUB_OUTPUT || : - - - name: sync kubernetes - id: sync_kubernetes - run: | - for v in $(cat version.tmp) - do - KUBERNETES_VERSION=$v QSCTL_ACCESS_KEY_ID=${{ secrets.QSCTL_ACCESS_KEY_ID }} QSCTL_SECRET_ACCESS_KEY=${{secrets.QSCTL_SECRET_ACCESS_KEY}} DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_PASSWORD=${{ secrets.DOCKERHUB_PASSWORD }} ALIYUNCS_USERNAME=${{ secrets.ALIYUNCS_USERNAME }} ALIYUNCS_PASSWORD=${{ secrets.ALIYUNCS_PASSWORD }} bash hack/sync-components.sh - done - echo "NEW_VERSION=`cat version.tmp | tr '\n' ', '`" >> $GITHUB_OUTPUT - rm -rf qsctl-config.yaml - rm -rf version.tmp - - make kk - chmod +x bin/kk - echo "## Kubernetes Versions(amd64/arm64)" > docs/kubernetes-versions.md - echo "| Version | Supported |" >> docs/kubernetes-versions.md - echo "|----------|--------------------|" >> docs/kubernetes-versions.md - bin/kk version --show-supported-k8s | ( while read version; do echo "| $version | :white_check_mark: |" >> docs/kubernetes-versions.md; done ) - - if: steps.get_new_version.outputs.UPDATE_VERSION == 'true' - - - name: Create Pull Request - uses: peter-evans/create-pull-request@v4 - with: - commit-message: Add new kubernetes version - committer: GitHub - signoff: false - branch: new_version - delete-branch: true - title: 'Add new kubernetes version' - body: | - Add kubernetes version: ${{ steps.sync_kubernetes.outputs.NEW_VERSION }} - - if: steps.get_new_version.outputs.UPDATE_VERSION == 'true' diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml deleted file mode 100644 index 17fdb961d..000000000 --- a/.github/workflows/release-drafter.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: Release Drafter - -on: - push: - branches: - - master - -jobs: - update_release_draft: - runs-on: ubuntu-latest - steps: - - uses: release-drafter/release-drafter@v5 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml deleted file mode 100644 index 19ac80a0d..000000000 --- a/.github/workflows/release.yaml +++ /dev/null @@ -1,46 +0,0 @@ -name: Release - -on: - push: - tags: - - '*' - -jobs: - build: - name: create draft release - runs-on: ubuntu-latest - steps: - - name: Set env - run: echo "RELEASE_TAG=${GITHUB_REF:10}" >> $GITHUB_ENV - - name: checkout code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Install go - uses: actions/setup-go@v3 - with: - go-version: '^1.19' - - name: generate release artifacts - run: | - make release - - name: Release - uses: softprops/action-gh-release@v1 - with: - draft: true - files: out/* - - - name: Get Version - id: get_version - run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT - - name: Synchronize artifacts to OSS - run: | - rm -rf qsctl_v2.4.3_linux_amd64.tar.gz - wget https://attack-on-titan.gd2.qingstor.com/qsctl/v2.4.3/qsctl_v2.4.3_linux_amd64.tar.gz - tar -zxvf qsctl_v2.4.3_linux_amd64.tar.gz - rm -rf qsctl_v2.4.3_linux_amd64.tar.gz - mv qsctl_v2.4.3_linux_amd64 /usr/local/bin/qsctl - echo "access_key_id: ${{secrets.KS_QSCTL_ACCESS_KEY_ID}}" > qsctl-config.yaml - echo "secret_access_key: ${{ secrets.KS_QSCTL_SECRET_ACCESS_KEY }}" >> qsctl-config.yaml - qsctl cp out/kubekey-${{ steps.get_version.outputs.VERSION }}-linux-amd64.tar.gz qs://kubernetes/kubekey/releases/download/${{ steps.get_version.outputs.VERSION }}/kubekey-${{ steps.get_version.outputs.VERSION }}-linux-amd64.tar.gz -c qsctl-config.yaml - qsctl cp out/kubekey-${{ steps.get_version.outputs.VERSION }}-linux-arm64.tar.gz qs://kubernetes/kubekey/releases/download/${{ steps.get_version.outputs.VERSION }}/kubekey-${{ steps.get_version.outputs.VERSION }}-linux-arm64.tar.gz -c qsctl-config.yaml - rm -rf qsctl-config.yaml diff --git a/.github/workflows/releaser.yaml b/.github/workflows/releaser.yaml new file mode 100644 index 000000000..fb6255631 --- /dev/null +++ b/.github/workflows/releaser.yaml @@ -0,0 +1,53 @@ +name: BuildReleaser + +on: + push: + tags: + - 'v4*' + +jobs: + goreleaser: + runs-on: ubuntu-latest + if: github.repository == 'kubesphere/kubekey' + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + fetch-depth: 0 + - name: Prepare + id: prepare + run: | + LDFLAGS=$(bash hack/version.sh) + VERSION=latest + if [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/} + fi + + echo "ldflags=${LDFLAGS}" >> "$GITHUB_OUTPUT" + echo "version=${VERSION}" >> "$GITHUB_OUTPUT" + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: 1.23.3 + + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v4 + with: + distribution: goreleaser + version: v2.5.1 + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + LDFLAGS: ${{ steps.prepare.outputs.ldflags }} + + - name: Upload Extra File + run: | + TAG=${{ steps.prepare.outputs.version }} + if [[ $tag =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + REGISTRY=docker.io/kubesphere TAG=$tag make generate + else + REGISTRY=docker.io/kubespheredev TAG=$tag make generate + fi + gh release upload "$TAG" config/capkk/release/* --clobber + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index b7fb3b29b..1cf278fd1 100644 --- a/.gitignore +++ b/.gitignore @@ -4,18 +4,11 @@ *.so *.dylib *.tmp -bin -hack/tools/bin +.DS_Store # Test binary, build with `go test -c` *.test -# E2E test templates -test/e2e/data/infrastructure-kubekey/v1beta1/cluster-template*.yaml - -# Output of the go coverage tool, specifically when used with LiteIDE -*.out - # IntelliJ .idea/ *.iml @@ -35,4 +28,8 @@ _artifacts # Used during parts of the build process. Files _should_ get cleaned up automatically. # This is also a good location for any temporary manfiests used during development -tmp \ No newline at end of file +tmp + +# Used by current object +/_output/ +dist/ diff --git a/.go-version b/.go-version new file mode 100644 index 000000000..ac1df3fce --- /dev/null +++ b/.go-version @@ -0,0 +1 @@ +1.23.3 diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 000000000..9e418fb92 --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,913 @@ +linters: + disable-all: true + enable: + - asasalint + - asciicheck + - bidichk + - bodyclose + - canonicalheader + - containedctx + - contextcheck + - copyloopvar + - cyclop + - decorder + - depguard + - dogsled +# - dupl + - dupword + - durationcheck +# - err113 + - errcheck + - errchkjson + - errname + - errorlint + # - exhaustive +# - exhaustruct + - copyloopvar + - fatcontext + - forbidigo + - forcetypeassert + - funlen +# - gci + - ginkgolinter + - gocheckcompilerdirectives +# - gochecknoglobals +# - gochecknoinits + - gochecksumtype + - gocognit + - goconst + - gocritic + - gocyclo + - godot + # - godox + - gofmt +# - gofumpt + - goheader + - goimports + # - gomoddirectives + - gomodguard + - goprintffuncname + - gosec + - gosimple + - gosmopolitan + - govet + - grouper + - importas + - inamedparam + - ineffassign + - interfacebloat + - intrange +# - ireturn +# - lll + - loggercheck + - maintidx + - makezero + - mirror +# - misspell +# - mnd +# - musttag + - nakedret + - nestif + # - nilerr + # - nilnil + # - nlreturn + - noctx + - nolintlint + # - nonamedreturns + - nosprintfhostport +# - paralleltest + - perfsprint + - prealloc + - predeclared + - promlinter + - protogetter + - reassign + - revive + - rowserrcheck + - sloglint + - spancheck + - sqlclosecheck + - staticcheck + - stylecheck + - tagalign +# - tagliatelle + - tenv + - testableexamples + - testifylint +# - testpackage + - thelper + - tparallel + - typecheck + - unconvert + - unparam + - unused + - usestdlibvars +# - varnamelen + - wastedassign + - whitespace +# - wrapcheck +# - wsl + - zerologlint + +linters-settings: + cyclop: + # The maximal code complexity to report. + # Default: 10 + max-complexity: 25 + # Should ignore tests. + # Default: false + skip-tests: true + depguard: + # Rules to apply. + # + # Variables: + # - File Variables + # you can still use and exclamation mark ! in front of a variable to say not to use it. + # Example !$test will match any file that is not a go test file. + # + # `$all` - matches all go files + # `$test` - matches all go test files + # + # - Package Variables + # + # `$gostd` - matches all of go's standard library (Pulled from `GOROOT`) + # + # Default: Only allow $gostd in all files. + rules: + # Name of a rule. + main: + # List of allowed packages. + allow: + - $gostd + - github.com/Masterminds/sprig + - github.com/cockroachdb/errors + - github.com/containerd/containerd/images + - github.com/emicklei/go-restful-openapi/v2 + - github.com/emicklei/go-restful/v3 + - github.com/evanphx/json-patch + - github.com/fsnotify/fsnotify + - github.com/go-git/go-git + - github.com/go-openapi/spec + - github.com/google/go-cmp/cmp + - github.com/google/gops + - github.com/kubesphere/kubekey + - github.com/opencontainers/image-spec + - github.com/pkg/sftp + - github.com/schollz/progressbar + - github.com/spf13 + - github.com/stretchr/testify + - k8s.io + - oras.land/oras-go + - sigs.k8s.io + forbidigo: + # Forbid the following identifiers (list of regexp). + # Default: ["^(fmt\\.Print(|f|ln)|print|println)$"] + forbid: + # Builtin function: + - ^print.*$ + # Optional message that gets included in error reports. +# - p: ^fmt\.Print.*$ +# msg: Do not commit print statements. +# # Alternatively, put messages at the end of the regex, surrounded by `(# )?` + # Escape any special characters. Those messages get included in error reports. +# - 'fmt\.Print.*(# Do not commit print statements\.)?' + # Forbid spew Dump, whether it is called as function or method. + # Depends on analyze-types below. + - ^spew\.(ConfigState\.)?Dump$ + # The package name might be ambiguous. + # The full import path can be used as additional criteria. + # Depends on analyze-types below. + - p: ^v1.Dump$ + pkg: ^example.com/pkg/api/v1$ + funlen: + # Checks the number of lines in a function. + # If lower than 0, disable the check. + # Default: 60 + lines: -1 + # Checks the number of statements in a function. + # If lower than 0, disable the check. + # Default: 40 + statements: -1 + # Ignore comments when counting lines. + # Default false + ignore-comments: true + gocognit: + min-complexity: 50 + gocritic: + enabled-tags: + - experimental + disabled-checks: + - appendAssign + - dupImport # https://github.com/go-critic/go-critic/issues/845 + - evalOrder + - ifElseChain + - octalLiteral + - regexpSimplify + - sloppyReassign + - truncateCmp + - typeDefFirst + - unnamedResult + - unnecessaryDefer + - whyNoLint + - wrapperFunc + - commentFormatting + - filepathJoin + # - rangeValCopy + # - hugeParam + godot: + # declarations - for top level declaration comments (default); + # toplevel - for top level comments; + # all - for all comments. + scope: toplevel + exclude: + - '^ \+.*' + - '^ ANCHOR.*' + # Check that each sentence ends with a period. + # Default: true + period: false + # Check that each sentence starts with a capital letter. + # Default: false + capital: false + gosec: + excludes: + - G106 # Deferring unsafe method "InsecureIgnoreHostKey" on type "\*ssh" + - G301 # Deferring unsafe method "MkdirAll" on type "\*os.File" + - G302 # Deferring unsafe method "Create" or "Open" on type "\*os.File" + - G304 # Deferring unsafe method "Create" or "Open" on type "\*os.File" + - G306 # Deferring unsafe method "WriteFile" on type "\*os.File" + - G307 # Deferring unsafe method "Close" on type "\*os.File" + - G108 # Profiling endpoint is automatically exposed on /debug/pprof + - G402 # Look for bad TLS connection settings + importas: + no-unaliased: true +# no-extra-aliases: true + alias: + # oci + - pkg: github.com/opencontainers/image-spec/specs-go/v1 + alias: imagev1 + # kubekey + - pkg: "github.com/kubesphere/kubekey/v4/pkg/const" + alias: _const + - pkg: "github.com/kubesphere/kubekey/api/capkk/infrastructure/v1beta1" + alias: capkkinfrav1beta1 + - pkg: "github.com/kubesphere/kubekey/api/core/v1" + alias: kkcorev1 + - pkg: "github.com/kubesphere/kubekey/api/core/v1alpha1" + alias: kkcorev1alpha1 + - pkg: "github.com/kubesphere/kubekey/api/project/v1" + alias: kkprojectv1 + # Kubernetes + - pkg: "k8s.io/api/batch/v1" + alias: batchv1 + - pkg: "k8s.io/api/coordination/v1" + alias: coordinationv1 + - pkg: "k8s.io/api/core/v1" + alias: corev1 + - pkg: "k8s.io/api/rbac/v1" + alias: rbacv1 + - pkg: "k8s.io/apimachinery/pkg/api/errors" + alias: apierrors + - pkg: "k8s.io/apimachinery/pkg/apis/meta/v1" + alias: metav1 + - pkg: "k8s.io/apimachinery/pkg/util/runtime" + alias: utilruntime + - pkg: "k8s.io/client-go/tools/cache" + alias: cgtoolscache + - pkg: "k8s.io/client-go/util/cert" + alias: cgutilcert + - pkg: "k8s.io/component-base/cli/flag" + alias: cliflag + - pkg: "k8s.io/utils/exec/testing" + alias: testingexec + - pkg: "k8s.io/utils/net" + alias: netutils + # Controller Runtime + - pkg: "sigs.k8s.io/controller-runtime" + alias: ctrl + - pkg: "sigs.k8s.io/controller-runtime/pkg/client" + alias: ctrlclient + - pkg: "sigs.k8s.io/controller-runtime/pkg/controller" + alias: ctrlcontroller + - pkg: "sigs.k8s.io/controller-runtime/pkg/finalizer" + alias: ctrlfinalizer + # cluster-api + - pkg: "sigs.k8s.io/cluster-api/api/v1beta1" + alias: clusterv1beta1 + - pkg: "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" + alias: kubeadmcpv1beta1 + - pkg: "sigs.k8s.io/cluster-api/util" + alias: clusterutil + # common + - pkg: "github.com/evanphx/json-patch" + alias: jsonpatch + nestif: + # Minimal complexity of if statements to report. + # Default: 5 + min-complexity: 20 + nolintlint: + allow-unused: false + require-specific: true + revive: + # Maximum number of open files at the same time. + # See https://github.com/mgechev/revive#command-line-flags + # Defaults to unlimited. +# max-open-files: 2048 + # When set to false, ignores files with "GENERATED" header, similar to golint. + # See https://github.com/mgechev/revive#available-rules for details. + # Default: false + ignore-generated-header: true + # Sets the default severity. + # See https://github.com/mgechev/revive#configuration + # Default: warning + severity: error + # Enable all available rules. + # Default: false + enable-all-rules: false + # Sets the default failure confidence. + # This means that linting errors with less than 0.8 confidence will be ignored. + # Default: 0.8 + confidence: 0.1 + rules: # v1.3.7 + # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#add-constant + - name: add-constant + severity: warning + disabled: true + exclude: [""] + arguments: + - maxLitCount: "3" + allowStrs: '""' + allowInts: "0,1,2" + allowFloats: "0.0,0.,1.0,1.,2.0,2." + # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#argument-limit + - name: argument-limit + severity: warning + disabled: true + exclude: [""] + arguments: [4] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#atomic + - name: atomic + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#banned-characters + - name: banned-characters + severity: warning + disabled: true + exclude: [""] + arguments: ["Ω", "Σ", "σ", "7"] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#bare-return + - name: bare-return + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#blank-imports + - name: blank-imports + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#bool-literal-in-expr + - name: bool-literal-in-expr + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#call-to-gc + - name: call-to-gc + severity: warning + disabled: true + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#cognitive-complexity + - name: cognitive-complexity + severity: warning + disabled: true + exclude: [""] + arguments: [7] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#comment-spacings + - name: comment-spacings + severity: warning + disabled: true + exclude: [""] + arguments: + - mypragma + - otherpragma +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#confusing-naming + - name: confusing-naming + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#confusing-results + - name: confusing-results + severity: warning + disabled: true + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#constant-logical-expr + - name: constant-logical-expr + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#context-as-argument + - name: context-as-argument + severity: warning + disabled: false + exclude: [""] +# arguments: +# - allowTypesBefore: "*testing.T,*github.com/user/repo/testing.Harness" +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#context-keys-type + - name: context-keys-type + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#cyclomatic + - name: cyclomatic + severity: warning + disabled: true + exclude: [""] + arguments: [3] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#datarace + - name: datarace + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#deep-exit + - name: deep-exit + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#defer + - name: defer + severity: warning + disabled: false + exclude: [""] + arguments: + - ["call-chain", "loop"] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#dot-imports + - name: dot-imports + severity: warning + disabled: false + exclude: [""] + arguments: [] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#duplicated-imports + - name: duplicated-imports + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#early-return + - name: early-return + severity: warning + disabled: false + exclude: [""] + arguments: + - "preserveScope" +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#empty-block + - name: empty-block + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#empty-lines + - name: empty-lines + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#enforce-map-style + - name: enforce-map-style + severity: warning + disabled: true + exclude: [""] + arguments: + - "make" +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#enforce-repeated-arg-type-style + - name: enforce-repeated-arg-type-style + severity: warning + disabled: true + exclude: [""] + arguments: + - "short" +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#enforce-slice-style + - name: enforce-slice-style + severity: warning + disabled: true + exclude: [""] + arguments: + - "make" +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#error-naming + - name: error-naming + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#error-return + - name: error-return + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#error-strings + - name: error-strings + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#errorf + - name: errorf + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#exported + - name: exported + severity: warning + disabled: false + exclude: [""] + arguments: + - "checkPrivateReceivers" + - "disableStutteringCheck" + - "sayRepetitiveInsteadOfStutters" +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#file-header + - name: file-header + severity: warning + disabled: true + exclude: [""] + arguments: + - This is the text that must appear at the top of source files. +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#flag-parameter + - name: flag-parameter + severity: warning + disabled: true + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#function-length + - name: function-length + severity: warning + disabled: true + exclude: [""] + arguments: [10, 0] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#function-result-limit + - name: function-result-limit + severity: warning + disabled: true + exclude: [""] + arguments: [3] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#get-return + - name: get-return + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#identical-branches + - name: identical-branches + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#if-return + - name: if-return + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#import-alias-naming + - name: import-alias-naming + severity: warning + disabled: true + exclude: [""] + arguments: + - "^[a-z][a-z0-9]{0,}$" +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#import-shadowing + - name: import-shadowing + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#imports-blocklist + - name: imports-blocklist + severity: warning + disabled: false + exclude: [""] + arguments: + - "crypto/md5" + - "crypto/sha1" +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#increment-decrement + - name: increment-decrement + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#indent-error-flow + - name: indent-error-flow + severity: warning + disabled: false + exclude: [""] + arguments: + - "preserveScope" +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#line-length-limit + - name: line-length-limit + severity: warning + disabled: true + exclude: [""] + arguments: [80] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#max-control-nesting + - name: max-control-nesting + severity: warning + disabled: true + exclude: [""] + arguments: [3] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#max-public-structs + - name: max-public-structs + severity: warning + disabled: true + exclude: [""] + arguments: [3] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#modifies-parameter + - name: modifies-parameter + severity: warning + disabled: true + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#modifies-value-receiver + - name: modifies-value-receiver + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#nested-structs + - name: nested-structs + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#optimize-operands-order + - name: optimize-operands-order + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#package-comments + - name: package-comments + severity: warning + disabled: true + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#range + - name: range + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#range-val-address + - name: range-val-address + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#range-val-in-closure + - name: range-val-in-closure + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#receiver-naming + - name: receiver-naming + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#redefines-builtin-id + - name: redefines-builtin-id + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#redundant-import-alias + - name: redundant-import-alias + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#string-format + - name: string-format + severity: warning + disabled: false + exclude: [""] + arguments: + - - 'core.WriteError[1].Message' + - '/^([^A-Z]|$)/' + - must not start with a capital letter + - - panic + - '/^[^\n]*$/' + - must not contain line breaks +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#string-of-int + - name: string-of-int + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#struct-tag + - name: struct-tag + severity: warning + disabled: true + exclude: [""] + arguments: + - "json,inline" + - "bson,outline,gnu" +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#superfluous-else + - name: superfluous-else + severity: warning + disabled: false + exclude: [""] + arguments: + - "preserveScope" +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#time-equal + - name: time-equal + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#time-naming + - name: time-naming + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unchecked-type-assertion + - name: unchecked-type-assertion + severity: warning + disabled: false + exclude: [""] + arguments: + - acceptIgnoredAssertionResult: true +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unconditional-recursion + - name: unconditional-recursion + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unexported-naming + - name: unexported-naming + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unexported-return + - name: unexported-return + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unhandled-error + - name: unhandled-error + severity: warning + disabled: false + exclude: [""] + arguments: + - "fmt.Printf" + - "fmt.Fprintf" + - "fmt.Fprint" + - "fmt.Println" + - "bytes.Buffer.WriteString" +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unnecessary-stmt + - name: unnecessary-stmt + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unreachable-code + - name: unreachable-code + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unused-parameter + - name: unused-parameter + severity: warning + disabled: true + exclude: [""] + arguments: + - allowRegex: "^_" +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unused-receiver + - name: unused-receiver + severity: warning + disabled: true + exclude: [""] + arguments: + - allowRegex: "^_" +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#use-any + - name: use-any + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#useless-break + - name: useless-break + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#var-declaration + - name: var-declaration + severity: warning + disabled: false + exclude: [""] +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#var-naming + - name: var-naming + severity: warning + disabled: false + exclude: [""] + arguments: + - ["ID"] # AllowList + - ["VM"] # DenyList + - - upperCaseConst: true # Extra parameter (upperCaseConst|skipPackageNameChecks) + skipPackageNameChecks: true +# # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#waitgroup-by-value + - name: waitgroup-by-value + severity: warning + disabled: false + exclude: [""] + stylecheck: + checks: + - -ST1000 # ignore package comment + wrapcheck: + # An array of strings that specify substrings of signatures to ignore. + # If this set, it will override the default set of ignored signatures. + # See https://github.com/tomarrell/wrapcheck#configuration for more information. + # Default: [".Errorf(", "errors.New(", "errors.Unwrap(", "errors.Join(", ".Wrap(", ".Wrapf(", ".WithMessage(", ".WithMessagef(", ".WithStack("] + ignoreSigs: + - .Errorf( + - errors.New( + - errors.Unwrap( + - errors.Join( + - .Wrap( + - .Wrapf( + - .WithMessage( + - .WithMessagef( + - .WithStack( + # An array of strings that specify regular expressions of signatures to ignore. + # Default: [] + ignoreSigRegexps: + - \.New.*Error\( + # An array of strings that specify globs of packages to ignore. + # Default: [] + ignorePackageGlobs: + - encoding/* + - github.com/pkg/* + # An array of strings that specify regular expressions of interfaces to ignore. + # Default: [] + ignoreInterfaceRegexps: + - ^(?i)c(?-i)ach(ing|e) +issues: + max-same-issues: 0 + max-issues-per-linter: 0 + # We are disabling default golangci exclusions because we want to help reviewers to focus on reviewing the most relevant + # changes in PRs and avoid nitpicking. + exclude-use-default: false + exclude-rules: + - linters: + - revive + text: "exported: exported method .*\\.(Reconcile|SetupWithManager|SetupWebhookWithManager) should have comment or be unexported" + - linters: + - errcheck + text: Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked + # Exclude revive's exported for certain packages and code, e.g. tests and fake. + - linters: + - revive + text: exported (method|function|type|const) (.+) should have comment or be unexported + source: (func|type).*Fake.* + - linters: + - revive + text: exported (method|function|type|const) (.+) should have comment or be unexported + path: fake_\.go + - linters: + - revive + text: exported (method|function|type|const) (.+) should have comment or be unexported + path: .*test/(providers|framework|e2e).*.go + - linters: + - errcheck + text: Error return value is not checked + path: _test\.go + - linters: + - errcheck + text: Error return value of (.+) is not checked + path: _test\.go + - linters: + - gosec + text: "G108: Profiling endpoint is automatically exposed on /debug/pprof" + - linters: + - godot + text: "Comment should end in a period" + path: "(.*)/(v1beta1|v1beta2)/(.*)types.go" + - linters: + - errcheck + text: Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked + # With Go 1.16, the new embed directive can be used with an un-named import, + # revive (previously, golint) only allows these to be imported in a main.go, which wouldn't work for us. + # This directive allows the embed package to be imported with an underscore everywhere. + - linters: + - revive + source: _ "embed" + # This directive allows the variable in defaults.go files to have underscore + - linters: + - revive + text: "var-naming: don't use underscores in Go names; func (.+) should be (.+)" + path: .*/defaults.go + # Disable unparam "always receives" which might not be really + # useful when building libraries. + - linters: + - unparam + text: always receives + # Dot imports for gomega or ginkgo are allowed + # within test files. + - path: _test\.go + text: should not use dot imports + - path: (framework|e2e)/.*.go + text: should not use dot imports + - path: _test\.go + text: cyclomatic complexity + - linters: + - unparam + text: (.+) - (`t`|`g`) is unused + - path: _test\.go + text: cyclomatic complexity + # Append should be able to assign to a different var/slice. + - linters: + - gocritic + text: "appendAssign: append result not assigned to the same slice" + # We don't care about defer in for loops in test files. + - linters: + - gocritic + text: "deferInLoop: Possible resource leak, 'defer' is called in the 'for' loop" + path: _test\.go + +run: + timeout: 10m + go: "1.23" + build-tags: + - builtin + - clusterapi + allow-parallel-runners: true diff --git a/.golangci.yml b/.golangci.yml deleted file mode 100644 index 511749985..000000000 --- a/.golangci.yml +++ /dev/null @@ -1,288 +0,0 @@ -linters: - disable-all: true - enable: - - asciicheck - - bodyclose - - containedctx - - deadcode - - depguard - - dogsled - - errcheck - - exportloopref - - gci - - goconst - - gocritic - - gofmt - - goimports - - goprintffuncname - - gosec - - gosimple - - govet - - importas - - ineffassign - - misspell - - nakedret - - nilerr - - noctx - - nolintlint - - prealloc - - predeclared - - revive - - rowserrcheck - - staticcheck - - structcheck - - stylecheck - - thelper - - typecheck - - unconvert - - unparam - - unused - - varcheck - - whitespace - -linters-settings: - godot: - # declarations - for top level declaration comments (default); - # toplevel - for top level comments; - # all - for all comments. - scope: toplevel - exclude: - - '^ \+.*' - - '^ ANCHOR.*' - ifshort: - # Maximum length of variable declaration measured in number of characters, after which linter won't suggest using short syntax. - max-decl-chars: 50 - gci: - local-prefixes: "github.com/kubesphere/kubekey" - importas: - no-unaliased: true - alias: - # Kubernetes - - pkg: k8s.io/api/core/v1 - alias: corev1 - - pkg: k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1 - alias: apiextensionsv1 - - pkg: k8s.io/apimachinery/pkg/apis/meta/v1 - alias: metav1 - - pkg: k8s.io/apimachinery/pkg/api/errors - alias: apierrors - - pkg: k8s.io/apimachinery/pkg/util/errors - alias: kerrors - # Controller Runtime - - pkg: sigs.k8s.io/controller-runtime - alias: ctrl - # CABPK - - pkg: sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha3 - alias: bootstrapv1alpha3 - - pkg: sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha4 - alias: bootstrapv1alpha4 - - pkg: sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1 - alias: bootstrapv1 - # KCP - - pkg: sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha3 - alias: controlplanev1alpha3 - - pkg: sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha4 - alias: controlplanev1alpha4 - - pkg: sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1 - alias: controlplanev1 - # CAPI - - pkg: sigs.k8s.io/cluster-api/api/v1alpha3 - alias: clusterv1alpha3 - - pkg: sigs.k8s.io/cluster-api/api/v1alpha4 - alias: clusterv1alpha4 - - pkg: sigs.k8s.io/cluster-api/api/v1beta1 - alias: clusterv1 - # CAPI exp - - pkg: sigs.k8s.io/cluster-api/exp/api/v1alpha3 - alias: expv1alpha3 - - pkg: sigs.k8s.io/cluster-api/exp/api/v1alpha4 - alias: expv1alpha4 - - pkg: sigs.k8s.io/cluster-api/exp/api/v1beta1 - alias: expv1 - # CAPI exp addons - - pkg: sigs.k8s.io/cluster-api/exp/addons/api/v1alpha3 - alias: addonsv1alpha3 - - pkg: sigs.k8s.io/cluster-api/exp/addons/api/v1alpha4 - alias: addonsv1alpha4 - - pkg: sigs.k8s.io/cluster-api/exp/addons/api/v1beta1 - alias: addonsv1 - # CAPI exp runtime - - pkg: sigs.k8s.io/cluster-api/exp/runtime/api/v1alpha1 - alias: runtimev1 - - pkg: sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1 - alias: runtimehooksv1 - - pkg: sigs.k8s.io/cluster-api/exp/runtime/controllers - alias: runtimecontrollers - - pkg: sigs.k8s.io/cluster-api/exp/runtime/catalog - alias: runtimecatalog - - pkg: sigs.k8s.io/cluster-api/internal/runtime/client - alias: runtimeclient - - pkg: sigs.k8s.io/cluster-api/internal/runtime/registry - alias: runtimeregistry - - pkg: sigs.k8s.io/cluster-api/internal/webhooks/runtime - alias: runtimewebhooks - # CAPKK - - pkg: github.com/kubesphere/kubekey/v3/api/v1beta1 - alias: infrav1 - - pkg: github.com/kubesphere/kubekey/v3/bootstrap/k3s/api/v1beta1 - alias: infrabootstrapv1 - - pkg: github.com/kubesphere/kubekey/v3/controlplane/k3s/api/v1beta1 - alias: infracontrolplanev1 - nolintlint: - allow-unused: false - allow-leading-space: false - require-specific: true - revive: - rules: - # The following rules are recommended https://github.com/mgechev/revive#recommended-configuration - - name: blank-imports - - name: context-as-argument - - name: context-keys-type - - name: dot-imports - - name: error-return - - name: error-strings - - name: error-naming - - name: exported - #- name: if-return # TODO This is a recommended rule with many findings which may require it's own pr. - - name: increment-decrement - - name: var-naming - - name: var-declaration - - name: package-comments - - name: range - - name: receiver-naming - - name: time-naming - - name: unexported-return - - name: indent-error-flow - - name: errorf - - name: empty-block - - name: superfluous-else - #- name: unused-parameter # TODO This is a recommended rule with many findings which may require it's own pr. - - name: unreachable-code - - name: redefines-builtin-id - # - # Rules in addition to the recommended configuration above. - # - - name: bool-literal-in-expr - - name: constant-logical-expr - gosec: - excludes: - - G307 # Deferring unsafe method "Close" on type "\*os.File" - - G108 # Profiling endpoint is automatically exposed on /debug/pprof - gocritic: - enabled-tags: - - experimental - disabled-checks: - - appendAssign - - dupImport # https://github.com/go-critic/go-critic/issues/845 - - evalOrder - - ifElseChain - - octalLiteral - - regexpSimplify - - sloppyReassign - - truncateCmp - - typeDefFirst - - unnamedResult - - unnecessaryDefer - - whyNoLint - - wrapperFunc - - commentFormatting - - filepathJoin - - rangeValCopy - - hugeParam -issues: - max-same-issues: 0 - max-issues-per-linter: 0 - # We are disabling default golangci exclusions because we want to help reviewers to focus on reviewing the most relevant - # changes in PRs and avoid nitpicking. - exclude-use-default: false - exclude-rules: - - linters: - - revive - text: "exported: exported method .*\\.(Reconcile|SetupWithManager|SetupWebhookWithManager) should have comment or be unexported" - - linters: - - errcheck - text: Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked - # Exclude revive's exported for certain packages and code, e.g. tests and fake. - - linters: - - revive - text: exported (method|function|type|const) (.+) should have comment or be unexported - source: (func|type).*Fake.* - - linters: - - revive - text: exported (method|function|type|const) (.+) should have comment or be unexported - path: fake_\.go - - linters: - - revive - text: exported (method|function|type|const) (.+) should have comment or be unexported - path: .*test/(providers|framework|e2e).*.go - - linters: - - errcheck - text: Error return value is not checked - path: _test\.go - - linters: - - errcheck - text: Error return value of (.+) is not checked - path: _test\.go - - linters: - - gosec - text: "G108: Profiling endpoint is automatically exposed on /debug/pprof" - - linters: - - godot - text: "Comment should end in a period" - path: "(.*)/(v1beta1|v1beta2)/(.*)types.go" - - linters: - - errcheck - text: Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked - # With Go 1.16, the new embed directive can be used with an un-named import, - # revive (previously, golint) only allows these to be imported in a main.go, which wouldn't work for us. - # This directive allows the embed package to be imported with an underscore everywhere. - - linters: - - revive - source: _ "embed" - # This directive allows the variable in defaults.go files to have underscore - - linters: - - revive - text: "var-naming: don't use underscores in Go names; func (.+) should be (.+)" - path: .*/defaults.go - # Disable unparam "always receives" which might not be really - # useful when building libraries. - - linters: - - unparam - text: always receives - # Dot imports for gomega or ginkgo are allowed - # within test files. - - path: _test\.go - text: should not use dot imports - - path: (framework|e2e)/.*.go - text: should not use dot imports - - path: _test\.go - text: cyclomatic complexity - - linters: - - unparam - text: (.+) - (`t`|`g`) is unused - - path: _test\.go - text: cyclomatic complexity - # Append should be able to assign to a different var/slice. - - linters: - - gocritic - text: "appendAssign: append result not assigned to the same slice" - # We don't care about defer in for loops in test files. - - linters: - - gocritic - text: "deferInLoop: Possible resource leak, 'defer' is called in the 'for' loop" - path: _test\.go - -run: - timeout: 10m - go: "1.19" - build-tags: - - tools - - e2e - - containers_image_openpgp - - exclude_graphdriver_devicemapper - - exclude_graphdriver_btrfs - skip-files: - - "zz_generated.*\\.go$" - - "vendored_openapi\\.go$" - - "cmd" - allow-parallel-runners: true diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 000000000..1637ad6d5 --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,40 @@ +version: 2 + +env: + - GO111MODULE=on + - CGO_ENABLED=0 + +before: + hooks: + - go mod tidy + +builds: + - main: ./cmd/kk/kubekey.go + goos: + - linux + - windows + - darwin + goarch: + - amd64 + - arm64 + binary: kk + ldflags: "{{ .Env.LDFLAGS }}" + flags: + - -trimpath + tags: + - builtin + +archives: + - format: tar.gz + name_template: "{{ .Binary }}_v{{ .Version }}_{{ .Os }}_{{ .Arch }}" + files: + - none* + +checksum: + name_template: "{{ .ProjectName }}_v{{ .Version }}_checksums.txt" + +release: + prerelease: auto + +universal_binaries: + - replace: true diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index cb04f2306..1b9073c18 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -1,4 +1,4 @@ -### Sincere gratitude goes to the following people for their contributions to KubeKey +### Sincere gratitude goes to the following people for their contributions to Pipeline Contributions of any kind are welcome! Thanks goes to these wonderful contributors, they made our project grow fast. @@ -111,7 +111,6 @@ Contributions of any kind are welcome! Thanks goes to these wonderful contributo wenwenxiong
wenwenxiong

💻 柏喵Sakura
柏喵Sakura

💻 cui fliter
cui fliter

📖 - 刘旭
刘旭

💻 diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 7584f3b73..000000000 --- a/Dockerfile +++ /dev/null @@ -1,61 +0,0 @@ -# Build architecture -ARG ARCH - -# Download dependencies -FROM alpine:3.11 as base_os_context - - -ENV OUTDIR=/out -RUN mkdir -p ${OUTDIR}/usr/local/bin/ - -WORKDIR /tmp - -RUN apk add --no-cache ca-certificates - -# Build the manager binary -FROM golang:1.19 as builder - -# Run this with docker build --build_arg $(go env GOPROXY) to override the goproxy -ARG goproxy=https://goproxy.cn,direct -ENV GOPROXY=$goproxy - -WORKDIR /workspace - -COPY go.mod go.mod -COPY go.sum go.sum - -# Cache deps before building and copying source so that we don't need to re-download as much -# and so that source changes don't invalidate our downloaded layer -RUN --mount=type=cache,target=/go/pkg/mod \ - go mod download - -# Copy the go source -COPY ./ ./ - -# Cache the go build into the the Go’s compiler cache folder so we take benefits of compiler caching across docker build calls -RUN --mount=type=cache,target=/root/.cache/go-build \ - --mount=type=cache,target=/go/pkg/mod \ - go build . - -# Build -ARG package=. -ARG ARCH -ARG LDFLAGS - -# Do not force rebuild of up-to-date packages (do not use -a) and use the compiler cache folder -RUN --mount=type=cache,target=/root/.cache/go-build \ - --mount=type=cache,target=/go/pkg/mod \ - CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \ - go build -ldflags "${LDFLAGS}" \ - -o manager ${package} - -FROM --platform=${ARCH} alpine:3.16 - -WORKDIR / - -RUN mkdir -p /var/lib/kubekey/rootfs - -COPY --from=base_os_context /out/ / -COPY --from=builder /workspace/manager . - -ENTRYPOINT ["/manager"] diff --git a/Makefile b/Makefile index 7d8d0767f..6afbc785e 100644 --- a/Makefile +++ b/Makefile @@ -6,9 +6,10 @@ SHELL:=/usr/bin/env bash # # Go. # -GO_VERSION ?= 1.19.2 +GO_VERSION ?= 1.23.3 GO_CONTAINER_IMAGE ?= docker.io/library/golang:$(GO_VERSION) - +GOARCH ?= $(shell go env GOARCH) +GOOS ?= $(shell go env GOOS) # Use GOPROXY environment variable if set GOPROXY := $(shell go env GOPROXY) ifeq ($(GOPROXY),) @@ -17,96 +18,111 @@ endif export GOPROXY # Active module mode, as we use go modules to manage dependencies -export GO111MODULE=on +#export GO111MODULE=on # This option is for running docker manifest command -export DOCKER_CLI_EXPERIMENTAL := enabled +#export DOCKER_CLI_EXPERIMENTAL := enabled # # Directories. # # Full directory of where the Makefile resides ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) -EXP_DIR := exp -BIN_DIR := bin +#EXP_DIR := exp + TEST_DIR := test TOOLS_DIR := hack/tools -TOOLS_BIN_DIR := $(abspath $(TOOLS_DIR)/$(BIN_DIR)) +#BIN_DIR := $(abspath $(TOOLS_DIR)/$(BIN_DIR)) E2E_FRAMEWORK_DIR := $(TEST_DIR)/framework -GO_INSTALL := ./scripts/go_install.sh +GO_INSTALL := ./hack/go_install.sh + +# output +OUTPUT_DIR := $(abspath $(ROOT_DIR)/_output) +OUTPUT_BIN_DIR := $(OUTPUT_DIR)/bin +OUTPUT_TOOLS_DIR := $(OUTPUT_DIR)/tools +#ARTIFACTS ?= ${OUTPUT_DIR}/_artifacts + +dirs := $(OUTPUT_DIR) $(OUTPUT_BIN_DIR) $(OUTPUT_TOOLS_DIR) + +$(foreach dir, $(dirs), \ + $(if $(shell [ -d $(dir) ] && echo 1 || echo 0),, \ + $(shell mkdir -p $(dir)) \ + ) \ +) -export PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH) +export PATH := $(abspath $(OUTPUT_BIN_DIR)):$(abspath $(OUTPUT_TOOLS_DIR)):$(PATH) # # Binaries. # # Note: Need to use abspath so we can invoke these from subdirectories -KUSTOMIZE_VER := v4.5.2 +KUSTOMIZE_VER := v5.5.0 KUSTOMIZE_BIN := kustomize -KUSTOMIZE := $(abspath $(TOOLS_BIN_DIR)/$(KUSTOMIZE_BIN)-$(KUSTOMIZE_VER)) -KUSTOMIZE_PKG := sigs.k8s.io/kustomize/kustomize/v4 +KUSTOMIZE := $(abspath $(OUTPUT_TOOLS_DIR)/$(KUSTOMIZE_BIN)-$(KUSTOMIZE_VER)) +KUSTOMIZE_PKG := sigs.k8s.io/kustomize/kustomize/v5 -SETUP_ENVTEST_VER := v0.0.0-20211110210527-619e6b92dab9 +SETUP_ENVTEST_VER := v0.0.0-20240521074430-fbb7d370bebc SETUP_ENVTEST_BIN := setup-envtest -SETUP_ENVTEST := $(abspath $(TOOLS_BIN_DIR)/$(SETUP_ENVTEST_BIN)-$(SETUP_ENVTEST_VER)) +SETUP_ENVTEST := $(abspath $(OUTPUT_TOOLS_DIR)/$(SETUP_ENVTEST_BIN)-$(SETUP_ENVTEST_VER)) SETUP_ENVTEST_PKG := sigs.k8s.io/controller-runtime/tools/setup-envtest -CONTROLLER_GEN_VER := v0.9.1 +CONTROLLER_GEN_VER := main CONTROLLER_GEN_BIN := controller-gen -CONTROLLER_GEN := $(abspath $(TOOLS_BIN_DIR)/$(CONTROLLER_GEN_BIN)-$(CONTROLLER_GEN_VER)) +CONTROLLER_GEN := $(abspath $(OUTPUT_TOOLS_DIR)/$(CONTROLLER_GEN_BIN)-$(CONTROLLER_GEN_VER)) CONTROLLER_GEN_PKG := sigs.k8s.io/controller-tools/cmd/controller-gen GOTESTSUM_VER := v1.6.4 GOTESTSUM_BIN := gotestsum -GOTESTSUM := $(abspath $(TOOLS_BIN_DIR)/$(GOTESTSUM_BIN)-$(GOTESTSUM_VER)) +GOTESTSUM := $(abspath $(OUTPUT_TOOLS_DIR)/$(GOTESTSUM_BIN)-$(GOTESTSUM_VER)) GOTESTSUM_PKG := gotest.tools/gotestsum HADOLINT_VER := v2.10.0 HADOLINT_FAILURE_THRESHOLD = warning +GOLANGCI_LINT_VER := $(shell cat .github/workflows/golangci-lint.yaml | grep [[:space:]]version | sed 's/.*version: //') GOLANGCI_LINT_BIN := golangci-lint -GOLANGCI_LINT := $(abspath $(TOOLS_BIN_DIR)/$(GOLANGCI_LINT_BIN)) +GOLANGCI_LINT := $(abspath $(OUTPUT_TOOLS_DIR)/$(GOLANGCI_LINT_BIN)-$(GOLANGCI_LINT_VER)) +GOLANGCI_LINT_PKG := github.com/golangci/golangci-lint/cmd/golangci-lint + +GORELEASER_VER := $(shell cat .github/workflows/releaser.yaml | grep [[:space:]]version | sed 's/.*version: //') +GORELEASER_BIN := goreleaser +GORELEASER := $(abspath $(OUTPUT_TOOLS_DIR)/$(GORELEASER_BIN)-$(GORELEASER_VER)) +GORELEASER_PKG := github.com/goreleaser/goreleaser/v2 + +# +# Docker. +# +DOCKERCMD ?= $(shell which docker) +DOCKER_BUILD_ENV = DOCKER_BUILDKIT=1 +DOCKER_BUILD ?= $(DOCKER_BUILD_ENV) $(DOCKERCMD) buildx build +PLATFORM ?= linux/amd64,linux/arm64 +DOCKER_OUT_TYPE ?= --push +DOCKER_PUSH ?= $(DOCKER_BUILD) --platform $(PLATFORM) $(DOCKER_OUT_TYPE) # Define Docker related variables. Releases should modify and double check these vars. REGISTRY ?= docker.io/kubespheredev -PROD_REGISTRY ?= docker.io/kubesphere # capkk -CAPKK_IMAGE_NAME ?= capkk-controller -CAPKK_CONTROLLER_IMG ?= $(REGISTRY)/$(CAPKK_IMAGE_NAME) - -# bootstrap -K3S_BOOTSTRAP_IMAGE_NAME ?= k3s-bootstrap-controller -K3S_BOOTSTRAP_CONTROLLER_IMG ?= $(REGISTRY)/$(K3S_BOOTSTRAP_IMAGE_NAME) - -# control plane -K3S_CONTROL_PLANE_IMAGE_NAME ?= k3s-control-plane-controller -K3S_CONTROL_PLANE_CONTROLLER_IMG ?= $(REGISTRY)/$(K3S_CONTROL_PLANE_IMAGE_NAME) +CAPKK_CONTROLLER_IMG_NAME ?= capkk-controller-manager +CAPKK_CONTROLLER_IMG ?= $(REGISTRY)/$(CAPKK_CONTROLLER_IMG_NAME) +# controller-manager +KK_CONTROLLER_IMG_NAME ?= kk-controller-manager +KK_CONTROLLER_IMG ?= $(REGISTRY)/$(KK_CONTROLLER_IMG_NAME) +# executor +KK_EXECUTOR_IMG_NAME ?= kk-executor +KK_EXECUTOR_IMG ?= $(REGISTRY)/$(KK_EXECUTOR_IMG_NAME) # It is set by Prow GIT_TAG, a git-based tag of the form vYYYYMMDD-hash, e.g., v20210120-v0.3.10-308-gc61521971 TAG ?= dev -ARCH ?= $(shell go env GOARCH) -ALL_ARCH = amd64 arm arm64 ppc64le s390x - -# Allow overriding the imagePullPolicy -PULL_POLICY ?= Always - -# Hosts running SELinux need :z added to volume mounts -SELINUX_ENABLED := $(shell cat /sys/fs/selinux/enforce 2> /dev/null || echo 0) - -ifeq ($(SELINUX_ENABLED),1) - DOCKER_VOL_OPTS?=:z -endif # Set build time variables including version details LDFLAGS := $(shell hack/version.sh) - # Set kk build tags -BUILDTAGS = exclude_graphdriver_devicemapper exclude_graphdriver_btrfs containers_image_openpgp - -.PHONY: all -all: test managers +#BUILDTAGS = exclude_graphdriver_devicemapper exclude_graphdriver_btrfs containers_image_openpgp +BUILDTAGS ?= builtin +#.PHONY: all +#all: test managers .PHONY: help help: ## Display this help. @@ -114,86 +130,53 @@ help: ## Display this help. ## -------------------------------------- ## Generate / Manifests -## -------------------------------------- +## --------------------------------------: ##@ generate: -ALL_GENERATE_MODULES = capkk k3s-bootstrap k3s-control-plane - .PHONY: generate -generate: ## Run all generate-manifests-*, generate-go-deepcopy-* targets - $(MAKE) generate-modules generate-manifests generate-go-deepcopy - -.PHONY: generate-manifests -generate-manifests: ## Run all generate-manifest-* targets - $(MAKE) $(addprefix generate-manifests-,$(ALL_GENERATE_MODULES)) - -.PHONY: generate-manifests-capkk -generate-manifests-capkk: $(CONTROLLER_GEN) $(KUSTOMIZE) ## Generate manifests e.g. CRD, RBAC etc. for core - $(MAKE) clean-generated-yaml SRC_DIRS="./config/crd/bases" - $(CONTROLLER_GEN) \ - paths=./api/... \ - paths=./controllers/... \ - crd:crdVersions=v1 \ - rbac:roleName=manager-role \ - output:crd:dir=./config/crd/bases \ - output:webhook:dir=./config/webhook \ - webhook - -.PHONY: generate-manifests-k3s-bootstrap -generate-manifests-k3s-bootstrap: $(CONTROLLER_GEN) $(KUSTOMIZE) ## Generate manifests e.g. CRD, RBAC etc. for core - $(MAKE) clean-generated-yaml SRC_DIRS="./bootstrap/k3s/config/crd/bases" - $(CONTROLLER_GEN) \ - paths=./bootstrap/k3s/api/... \ - paths=./bootstrap/k3s/controllers/... \ - crd:crdVersions=v1 \ - rbac:roleName=manager-role \ - output:crd:dir=./bootstrap/k3s/config/crd/bases \ - output:rbac:dir=./bootstrap/k3s/config/rbac \ - output:webhook:dir=./bootstrap/k3s/config/webhook \ - webhook - -.PHONY: generate-manifests-k3s-control-plane -generate-manifests-k3s-control-plane: $(CONTROLLER_GEN) $(KUSTOMIZE) ## Generate manifests e.g. CRD, RBAC etc. for core - $(MAKE) clean-generated-yaml SRC_DIRS="./controlplane/k3s/config/crd/bases" - $(CONTROLLER_GEN) \ - paths=./controlplane/k3s/api/... \ - paths=./controlplane/k3s/controllers/... \ - crd:crdVersions=v1 \ - rbac:roleName=manager-role \ - output:crd:dir=./controlplane/k3s/config/crd/bases \ - output:rbac:dir=./controlplane/k3s/config/rbac \ - output:webhook:dir=./controlplane/k3s/config/webhook \ - webhook +generate: generate-go-deepcopy generate-manifests-kubekey generate-manifests-capkk generate-modules generate-goimports ## Run all generate-manifests-*, generate-go-deepcopy-* targets .PHONY: generate-go-deepcopy -generate-go-deepcopy: ## Run all generate-go-deepcopy-* targets - $(MAKE) $(addprefix generate-go-deepcopy-,$(ALL_GENERATE_MODULES)) - -.PHONY: generate-go-deepcopy-capkk -generate-go-deepcopy-capkk: $(CONTROLLER_GEN) ## Generate deepcopy go code for capkk - $(MAKE) clean-generated-deepcopy SRC_DIRS="./api" - $(CONTROLLER_GEN) \ +generate-go-deepcopy: $(CONTROLLER_GEN) ## Generate deepcopy object + $(MAKE) clean-generated-deepcopy SRC_DIRS="./api/" + @$(CONTROLLER_GEN) \ object:headerFile=./hack/boilerplate.go.txt \ - paths=./api/... \ + paths=./api/... -.PHONY: generate-go-deepcopy-k3s-bootstrap -generate-go-deepcopy-k3s-bootstrap: $(CONTROLLER_GEN) ## Generate deepcopy go code for k3s-bootstrap - $(MAKE) clean-generated-deepcopy SRC_DIRS="./bootstrap/k3s/api" - $(CONTROLLER_GEN) \ - object:headerFile=./hack/boilerplate.go.txt \ - paths=./bootstrap/k3s/api/... \ +.PHONY: generate-manifests-kubekey +generate-manifests-kubekey: $(CONTROLLER_GEN) clean-crds-kubekey ## Generate kubekey manifests e.g. CRD, RBAC etc. + @$(CONTROLLER_GEN) \ + paths=./api/core/... \ + crd output:crd:dir=./config/kubekey/crds/ + @$(CONTROLLER_GEN) \ + paths=./pkg/controllers/core/... \ + rbac:roleName=kubekey output:rbac:dir=./config/kubekey/templates/ -.PHONY: generate-go-deepcopy-k3s-control-plane -generate-go-deepcopy-k3s-control-plane: $(CONTROLLER_GEN) ## Generate deepcopy go code for k3s-control-plane - $(MAKE) clean-generated-deepcopy SRC_DIRS="./controlplane/k3s/api" - $(CONTROLLER_GEN) \ - object:headerFile=./hack/boilerplate.go.txt \ - paths=./controlplane/k3s/api/... \ +.PHONY: generate-manifests-capkk +generate-manifests-capkk: $(CONTROLLER_GEN) $(KUSTOMIZE) clean-crds-capkk ## Generate capkk manifests e.g. CRD, RBAC etc. + @$(CONTROLLER_GEN) \ + paths=./api/capkk/... \ + crd \ + output:crd:dir=./config/capkk/crds/ + @$(CONTROLLER_GEN) \ + paths=./pkg/controllers/... \ + rbac:roleName=capkk output:rbac:dir=./config/capkk/rbac \ + webhook output:webhook:dir=./config/capkk/webhook + @cp ./config/kubekey/crds/* ./config/capkk/crds/ + @cd config/capkk && $(KUSTOMIZE) edit set image capkk-controller-manager-image=$(CAPKK_CONTROLLER_IMG):$(TAG) kk-controller-manager-image=$(KK_CONTROLLER_IMG):$(TAG) + @$(KUSTOMIZE) build config/capkk | \ + yq eval '.metadata |= select(.name == "default-capkk") *+ {"annotations": {"cert-manager.io/inject-ca-from": "capkk-system/capkk-serving-cert"}}' | \ + yq eval '.spec.template.spec.containers[] |= (select(.name == "controller-manager") | .env[] |= (select(.name == "EXECUTOR_IMAGE") | .value = "$(KK_EXECUTOR_IMG):$(TAG)"))' \ + > config/capkk/release/infrastructure-components.yaml .PHONY: generate-modules generate-modules: ## Run go mod tidy to ensure modules are up to date - go mod tidy + @cd api && go mod tidy + @go mod tidy +.PHONY: generate-goimports +generate-goimports: ## Format all import, `goimports` is required. + @hack/update-goimports.sh ## -------------------------------------- ## Lint / Verify @@ -203,20 +186,20 @@ generate-modules: ## Run go mod tidy to ensure modules are up to date .PHONY: lint lint: $(GOLANGCI_LINT) ## Lint the codebase - $(GOLANGCI_LINT) run -v $(GOLANGCI_LINT_EXTRA_ARGS) - cd $(TEST_DIR); $(GOLANGCI_LINT) run -v $(GOLANGCI_LINT_EXTRA_ARGS) - cd $(TOOLS_DIR); $(GOLANGCI_LINT) run -v $(GOLANGCI_LINT_EXTRA_ARGS) - ./scripts/ci-lint-dockerfiles.sh $(HADOLINT_VER) $(HADOLINT_FAILURE_THRESHOLD) + @$(GOLANGCI_LINT) run -v $(GOLANGCI_LINT_EXTRA_ARGS) + @cd $(TEST_DIR); $(GOLANGCI_LINT) run -v $(GOLANGCI_LINT_EXTRA_ARGS) + +.PHONY: verify-dockerfiles +verify-dockerfiles: + @./hack/ci-lint-dockerfiles.sh $(HADOLINT_VER) $(HADOLINT_FAILURE_THRESHOLD) -.PHONY: lint-dockerfiles -lint-dockerfiles: - ./scripts/ci-lint-dockerfiles.sh $(HADOLINT_VER) $(HADOLINT_FAILURE_THRESHOLD) +ALL_VERIFY_CHECKS ?= modules gen goimports releaser .PHONY: verify -verify: $(addprefix verify-,$(ALL_VERIFY_CHECKS)) lint-dockerfiles ## Run all verify-* targets +verify: $(addprefix verify-,$(ALL_VERIFY_CHECKS)) ## Run all verify-* targets .PHONY: verify-modules -verify-modules: generate-modules ## Verify go modules are up to date +verify-modules: ## Verify go modules are up to date @if !(git diff --quiet HEAD -- go.sum go.mod $(TOOLS_DIR)/go.mod $(TOOLS_DIR)/go.sum $(TEST_DIR)/go.mod $(TEST_DIR)/go.sum); then \ git diff; \ echo "go module files are out of date"; exit 1; \ @@ -227,12 +210,20 @@ verify-modules: generate-modules ## Verify go modules are up to date fi .PHONY: verify-gen -verify-gen: generate ## Verify go generated files are up to date +verify-gen: ## Verify go generated files are up to date @if !(git diff --quiet HEAD); then \ git diff; \ echo "generated files are out of date, run make generate"; exit 1; \ fi +.PHONY: verify-goimports +verify-goimports: ## Verify go imports + @hack/verify-goimports.sh + +.PHONY: verify-releaser +verify-releaser: $(GORELEASER) ## Verify goreleaser + @$(GORELEASER) check + ## -------------------------------------- ## Binaries ## -------------------------------------- @@ -240,58 +231,39 @@ verify-gen: generate ## Verify go generated files are up to date ##@ build: .PHONY: kk -kk: - CGO_ENABLED=0 go build -trimpath -tags "$(BUILDTAGS)" -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/kk github.com/kubesphere/kubekey/v3/cmd/kk; - -ALL_MANAGERS = capkk k3s-bootstrap k3s-control-plane - -.PHONY: managers -managers: $(addprefix manager-,$(ALL_MANAGERS)) ## Run all manager-* targets - -.PHONY: manager-capkk -manager-capkk: ## Build the capkk manager binary into the ./bin folder - go build -trimpath -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/manager github.com/kubesphere/kubekey/v3 - -.PHONY: manager-k3s-bootstrap -manager-k3s-bootstrap: ## Build the k3s bootstrap manager binary into the ./bin folder - go build -trimpath -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/k3s-bootstrap-manager github.com/kubesphere/kubekey/v3/bootstrap/k3s - -.PHONY: manager-k3s-control-plane -manager-k3s-control-plane: ## Build the k3s control plane manager binary into the ./bin folder - go build -trimpath -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/k3s-control-plane-manager github.com/kubesphere/kubekey/v3/controlplane/k3s - -.PHONY: docker-pull-prerequisites -docker-pull-prerequisites: - docker pull docker.io/docker/dockerfile:1.4 - docker pull $(GO_CONTAINER_IMAGE) - -.PHONY: docker-build-all -docker-build-all: $(addprefix docker-build-,$(ALL_ARCH)) ## Build docker images for all architectures - -docker-build-%: - $(MAKE) ARCH=$* docker-build - -ALL_DOCKER_BUILD = capkk k3s-bootstrap k3s-control-plane - -.PHONY: docker-build -docker-build: docker-pull-prerequisites ## Run docker-build-* targets for all providers - $(MAKE) ARCH=$(ARCH) $(addprefix docker-build-,$(ALL_DOCKER_BUILD)) - -.PHONY: docker-build-capkk -docker-build-capkk: ## Build the docker image for capkk - DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg ldflags="$(LDFLAGS)" . -t $(CAPKK_CONTROLLER_IMG)-$(ARCH):$(TAG) - -.PHONY: docker-build-k3s-bootstrap -docker-build-k3s-bootstrap: ## Build the docker image for k3s bootstrap controller manager - DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg package=./bootstrap/k3s --build-arg ldflags="$(LDFLAGS)" . -t $(K3S_BOOTSTRAP_CONTROLLER_IMG)-$(ARCH):$(TAG) - -.PHONY: docker-build-k3s-control-plane -docker-build-k3s-control-plane: ## Build the docker image for k3s control plane controller manager - DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg package=./controlplane/k3s --build-arg ldflags="$(LDFLAGS)" . -t $(K3S_CONTROL_PLANE_CONTROLLER_IMG)-$(ARCH):$(TAG) - -.PHONY: docker-build-e2e -docker-build-e2e: ## Build the docker image for capkk - $(MAKE) docker-build REGISTRY=docker.io/kubespheredev PULL_POLICY=IfNotPresent TAG=e2e +kk: ## build kk binary + @CGO_ENABLED=0 GOARCH=$(GOARCH) GOOS=$(GOOS) go build -trimpath -tags "$(BUILDTAGS)" -ldflags "$(LDFLAGS)" -o $(OUTPUT_BIN_DIR)/kk cmd/kk/kubekey.go + +.PHONY: kk-releaser +kk-releaser: $(GORELEASER) ## build releaser in dist. it will show in https://github.com/kubesphere/kubekey/releases + @LDFLAGS=$(bash ./hack/version.sh) $(GORELEASER) release --clean --skip validate --skip publish + +.PHONY: docker-push ## build and push all images +docker-push: docker-push-kk-executor docker-push-kk-controller-manager docker-push-capkk-controller-manager + +.PHONY: docker-push-kk-executor +docker-push-kk-executor: ## Build the docker image for kk-executor + @$(DOCKER_PUSH) \ + --build-arg builder_image=$(GO_CONTAINER_IMAGE) \ + --build-arg goproxy=$(GOPROXY) \ + --build-arg ldflags="$(LDFLAGS)" --build-arg build_tags="" \ + -f build/kk/Dockerfile -t $(KK_EXECUTOR_IMG):$(TAG) . + +.PHONY: docker-push-kk-controller-manager +docker-push-kk-controller-manager: ## Build the docker image for kk-controller-manager + @$(DOCKER_PUSH) \ + --build-arg builder_image=$(GO_CONTAINER_IMAGE) \ + --build-arg goproxy=$(GOPROXY) \ + --build-arg ldflags="$(LDFLAGS)" --build-arg build_tags="builtin" \ + -f build/controller-manager/Dockerfile -t $(KK_CONTROLLER_IMG):$(TAG) . + +.PHONY: docker-push-capkk-controller-manager +docker-push-capkk-controller-manager: ## Build the docker image for capkk-controller-manager + @$(DOCKER_PUSH) \ + --build-arg builder_image=$(GO_CONTAINER_IMAGE) \ + --build-arg goproxy=$(GOPROXY) \ + --build-arg ldflags="$(LDFLAGS)" --build-arg build_tags="clusterapi" \ + -f build/controller-manager/Dockerfile -t $(CAPKK_CONTROLLER_IMG):$(TAG) . ## -------------------------------------- ## Deployment @@ -299,29 +271,9 @@ docker-build-e2e: ## Build the docker image for capkk ##@ deployment -ifndef ignore-not-found - ignore-not-found = false -endif - -.PHONY: install -install: generate $(KUSTOMIZE) ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/crd | kubectl apply -f - - -.PHONY: uninstall -uninstall: generate $(KUSTOMIZE) ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - - -.PHONY: deploy -deploy: generate $(KUSTOMIZE) ## Deploy controller to the K8s cluster specified in ~/.kube/config. - $(MAKE) set-manifest-image \ - MANIFEST_IMG=$(REGISTRY)/$(CAPKK_IMAGE_NAME)-$(ARCH) MANIFEST_TAG=$(TAG) \ - TARGET_RESOURCE="./config/default/manager_image_patch.yaml" - cd config/manager - $(KUSTOMIZE) build config/default | kubectl apply -f - - -.PHONY: undeploy -undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f - +.PHONY: helm-package +helm-package: ## Helm-package. + @helm package config/helm -d $(OUTPUT_DIR) ## -------------------------------------- ## Testing @@ -329,187 +281,13 @@ undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/confi ##@ test: -ARTIFACTS ?= ${ROOT_DIR}/_artifacts - -ifeq ($(shell go env GOOS),darwin) # Use the darwin/amd64 binary until an arm64 version is available - KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use --use-env -p path --arch amd64 $(KUBEBUILDER_ENVTEST_KUBERNETES_VERSION)) -else - KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use --use-env -p path $(KUBEBUILDER_ENVTEST_KUBERNETES_VERSION)) -endif - .PHONY: test test: $(SETUP_ENVTEST) ## Run unit and integration tests - KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test ./... $(TEST_ARGS) + @KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test ./... $(TEST_ARGS) .PHONY: test-verbose test-verbose: ## Run unit and integration tests with verbose flag - $(MAKE) test TEST_ARGS="$(TEST_ARGS) -v" - -.PHONY: test-junit -test-junit: $(SETUP_ENVTEST) $(GOTESTSUM) ## Run unit and integration tests and generate a junit report - set +o errexit; (KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test -json ./... $(TEST_ARGS); echo $$? > $(ARTIFACTS)/junit.exitcode) | tee $(ARTIFACTS)/junit.stdout - $(GOTESTSUM) --junitfile $(ARTIFACTS)/junit.xml --raw-command cat $(ARTIFACTS)/junit.stdout - exit $$(cat $(ARTIFACTS)/junit.exitcode) - -.PHONY: test-cover -test-cover: ## Run unit and integration tests and generate a coverage report - $(MAKE) test TEST_ARGS="$(TEST_ARGS) -coverprofile=out/coverage.out" - go tool cover -func=out/coverage.out -o out/coverage.txt - go tool cover -html=out/coverage.out -o out/coverage.html - -.PHONY: test-e2e -test-e2e: ## Run e2e tests - $(MAKE) -C $(TEST_DIR)/e2e run - -.PHONY: test-e2e-k3s -test-e2e-k3s: ## Run e2e tests - $(MAKE) -C $(TEST_DIR)/e2e run-k3s - -## -------------------------------------- -## Release -## -------------------------------------- - -##@ release: - -## latest git tag for the commit, e.g., v0.3.10 -RELEASE_TAG ?= $(shell git describe --abbrev=0 2>/dev/null) -ifneq (,$(findstring -,$(RELEASE_TAG))) - PRE_RELEASE=true -endif -# the previous release tag, e.g., v0.3.9, excluding pre-release tags -PREVIOUS_TAG ?= $(shell git tag -l | grep -E "^v[0-9]+\.[0-9]+\.[0-9]+$$" | sort -V | grep -B1 $(RELEASE_TAG) | head -n 1 2>/dev/null) -RELEASE_DIR := out - -$(RELEASE_DIR): - mkdir -p $(RELEASE_DIR)/ - -.PHONY: release -release: clean-release ## Build and push container images using the latest git tag for the commit - @if [ -z "${RELEASE_TAG}" ]; then echo "RELEASE_TAG is not set"; exit 1; fi - @if ! [ -z "$$(git status --porcelain)" ]; then echo "Your local git repository contains uncommitted changes, use git clean before proceeding."; exit 1; fi - git checkout "${RELEASE_TAG}" - ## Build binaries first. - GIT_VERSION=$(RELEASE_TAG) $(MAKE) release-binaries - # Set the manifest image to the production bucket. - $(MAKE) manifest-modification REGISTRY=$(PROD_REGISTRY) - ## Build the manifests - $(MAKE) release-manifests - ## Build the templates - $(MAKE) release-templates - ## Clean the git artifacts modified in the release process - $(MAKE) clean-release-git - -release-binaries: ## Build the binaries to publish with a release - RELEASE_BINARY=./cmd/kk GOOS=linux GOARCH=amd64 $(MAKE) release-binary - RELEASE_BINARY=./cmd/kk GOOS=linux GOARCH=amd64 $(MAKE) release-archive - RELEASE_BINARY=./cmd/kk GOOS=linux GOARCH=arm64 $(MAKE) release-binary - RELEASE_BINARY=./cmd/kk GOOS=linux GOARCH=arm64 $(MAKE) release-archive - RELEASE_BINARY=./cmd/kk GOOS=darwin GOARCH=amd64 $(MAKE) release-binary - RELEASE_BINARY=./cmd/kk GOOS=darwin GOARCH=amd64 $(MAKE) release-archive - RELEASE_BINARY=./cmd/kk GOOS=darwin GOARCH=arm64 $(MAKE) release-binary - RELEASE_BINARY=./cmd/kk GOOS=darwin GOARCH=arm64 $(MAKE) release-archive - -release-binary: $(RELEASE_DIR) - docker run \ - --rm \ - -e CGO_ENABLED=0 \ - -e GOOS=$(GOOS) \ - -e GOARCH=$(GOARCH) \ - -e GOPROXY=$(GOPROXY) \ - -v "$$(pwd):/workspace$(DOCKER_VOL_OPTS)" \ - -w /workspace \ - golang:$(GO_VERSION) \ - go build -a -trimpath -tags "$(BUILDTAGS)" -ldflags "$(LDFLAGS) -extldflags '-static'" \ - -o $(RELEASE_DIR)/$(notdir $(RELEASE_BINARY)) $(RELEASE_BINARY) - -release-archive: $(RELEASE_DIR) - tar -czf $(RELEASE_DIR)/kubekey-$(RELEASE_TAG)-$(GOOS)-$(GOARCH).tar.gz -C $(RELEASE_DIR)/ $(notdir $(RELEASE_BINARY)) - rm -rf $(RELEASE_DIR)/$(notdir $(RELEASE_BINARY)) - -.PHONY: manifest-modification -manifest-modification: # Set the manifest images to the staging/production bucket. - $(MAKE) set-manifest-image \ - MANIFEST_IMG=$(REGISTRY)/$(CAPKK_IMAGE_NAME) MANIFEST_TAG=$(RELEASE_TAG) \ - TARGET_RESOURCE="./config/default/manager_image_patch.yaml" - $(MAKE) set-manifest-image \ - MANIFEST_IMG=$(REGISTRY)/$(K3S_BOOTSTRAP_IMAGE_NAME) MANIFEST_TAG=$(RELEASE_TAG) \ - TARGET_RESOURCE="./bootstrap/k3s/config/default/manager_image_patch.yaml" - $(MAKE) set-manifest-image \ - MANIFEST_IMG=$(REGISTRY)/$(K3S_CONTROL_PLANE_IMAGE_NAME) MANIFEST_TAG=$(RELEASE_TAG) \ - TARGET_RESOURCE="./controlplane/k3s/config/default/manager_image_patch.yaml" - $(MAKE) set-manifest-pull-policy PULL_POLICY=IfNotPresent TARGET_RESOURCE="./config/default/manager_pull_policy.yaml" - $(MAKE) set-manifest-pull-policy PULL_POLICY=IfNotPresent TARGET_RESOURCE="./bootstrap/k3s/config/default/manager_pull_policy.yaml" - $(MAKE) set-manifest-pull-policy PULL_POLICY=IfNotPresent TARGET_RESOURCE="./controlplane/k3s/config/default/manager_pull_policy.yaml" - -.PHONY: release-manifests -release-manifests: $(RELEASE_DIR) $(KUSTOMIZE) ## Build the manifests to publish with a release - # Build capkk-components. - $(KUSTOMIZE) build config/default > $(RELEASE_DIR)/infrastructure-components.yaml - # Build bootstrap-components. - $(KUSTOMIZE) build bootstrap/k3s/config/default > $(RELEASE_DIR)/bootstrap-components.yaml - # Build control-plane-components. - $(KUSTOMIZE) build controlplane/k3s/config/default > $(RELEASE_DIR)/control-plane-components.yaml - - # Add metadata to the release artifacts - cp metadata.yaml $(RELEASE_DIR)/metadata.yaml - -.PHONY: release-templates -release-templates: $(RELEASE_DIR) ## Generate release templates - cp templates/cluster-template*.yaml $(RELEASE_DIR)/ - -.PHONY: release-prod -release-prod: ## Build and push container images to the prod - REGISTRY=$(PROD_REGISTRY) TAG=$(RELEASE_TAG) $(MAKE) docker-build-all docker-push-all - -## -------------------------------------- -## Docker -## -------------------------------------- - -.PHONY: docker-push-all -docker-push-all: $(addprefix docker-push-,$(ALL_ARCH)) ## Push the docker images to be included in the release for all architectures + related multiarch manifests - $(MAKE) docker-push-manifest-capkk - $(MAKE) docker-push-manifest-k3s-bootstrap - $(MAKE) docker-push-manifest-k3s-control-plane - -docker-push-%: - $(MAKE) ARCH=$* docker-push - -.PHONY: docker-push -docker-push: ## Push the docker images - docker push $(CAPKK_CONTROLLER_IMG)-$(ARCH):$(TAG) - docker push $(K3S_BOOTSTRAP_CONTROLLER_IMG)-$(ARCH):$(TAG) - docker push $(K3S_CONTROL_PLANE_CONTROLLER_IMG)-$(ARCH):$(TAG) - -.PHONY: docker-push-manifest-capkk -docker-push-manifest-capkk: ## Push the multiarch manifest for the capkk docker images - ## Minimum docker version 18.06.0 is required for creating and pushing manifest images. - docker manifest create --amend $(CAPKK_CONTROLLER_IMG):$(TAG) $(shell echo $(ALL_ARCH) | sed -e "s~[^ ]*~$(CAPKK_CONTROLLER_IMG)\-&:$(TAG)~g") - @for arch in $(ALL_ARCH); do docker manifest annotate --arch $${arch} ${CAPKK_CONTROLLER_IMG}:${TAG} ${CAPKK_CONTROLLER_IMG}-$${arch}:${TAG}; done - docker manifest push --purge $(CAPKK_CONTROLLER_IMG):$(TAG) - -.PHONY: docker-push-manifest-k3s-bootstrap -docker-push-manifest-k3s-bootstrap: ## Push the multiarch manifest for the k3s bootstrap docker images - ## Minimum docker version 18.06.0 is required for creating and pushing manifest images. - docker manifest create --amend $(K3S_BOOTSTRAP_CONTROLLER_IMG):$(TAG) $(shell echo $(ALL_ARCH) | sed -e "s~[^ ]*~$(K3S_BOOTSTRAP_CONTROLLER_IMG)\-&:$(TAG)~g") - @for arch in $(ALL_ARCH); do docker manifest annotate --arch $${arch} ${K3S_BOOTSTRAP_CONTROLLER_IMG}:${TAG} ${K3S_BOOTSTRAP_CONTROLLER_IMG}-$${arch}:${TAG}; done - docker manifest push --purge $(K3S_BOOTSTRAP_CONTROLLER_IMG):$(TAG) - -.PHONY: docker-push-manifest-k3s-control-plane -docker-push-manifest-k3s-control-plane: ## Push the multiarch manifest for the k3s control plane docker images - ## Minimum docker version 18.06.0 is required for creating and pushing manifest images. - docker manifest create --amend $(K3S_CONTROL_PLANE_CONTROLLER_IMG):$(TAG) $(shell echo $(ALL_ARCH) | sed -e "s~[^ ]*~$(K3S_CONTROL_PLANE_CONTROLLER_IMG)\-&:$(TAG)~g") - @for arch in $(ALL_ARCH); do docker manifest annotate --arch $${arch} ${K3S_CONTROL_PLANE_CONTROLLER_IMG}:${TAG} ${K3S_CONTROL_PLANE_CONTROLLER_IMG}-$${arch}:${TAG}; done - docker manifest push --purge $(K3S_CONTROL_PLANE_CONTROLLER_IMG):$(TAG) - -.PHONY: set-manifest-pull-policy -set-manifest-pull-policy: - $(info Updating kustomize pull policy file for manager resources) - sed -i'' -e 's@imagePullPolicy: .*@imagePullPolicy: '"$(PULL_POLICY)"'@' $(TARGET_RESOURCE) - -.PHONY: set-manifest-image -set-manifest-image: - $(info Updating kustomize image patch file for manager resource) - sed -i'' -e 's@image: .*@image: '"${MANIFEST_IMG}:$(MANIFEST_TAG)"'@' $(TARGET_RESOURCE) + @$(MAKE) test TEST_ARGS="$(TEST_ARGS) -v" ## -------------------------------------- ## Cleanup / Verification @@ -518,29 +296,31 @@ set-manifest-image: ##@ clean: .PHONY: clean -clean: ## Remove all generated files - $(MAKE) clean-bin +clean: clean-output clean-generated-deepcopy clean-crds-kubekey clean-crds-capkk ## Remove all generated files -.PHONY: clean-bin -clean-bin: ## Remove all generated binaries - rm -rf $(BIN_DIR) - rm -rf $(TOOLS_BIN_DIR) +.PHONY: clean-output +clean-output: ## Remove all generated binaries + @rm -rf $(OUTPUT_DIR) -.PHONY: clean-release -clean-release: ## Remove the release folder - rm -rf $(RELEASE_DIR) +.PHONY: clean-crds-kubekey +clean-crds-kubekey: ## Remove the generated crds for kubekey + @rm -rf ./config/kubekey/crds -.PHONY: clean-release-git -clean-release-git: ## Restores the git files usually modified during a release - git restore ./*manager_image_patch.yaml ./*manager_pull_policy.yaml - -.PHONY: clean-generated-yaml -clean-generated-yaml: ## Remove files generated by conversion-gen from the mentioned dirs. Example SRC_DIRS="./api/v1beta1" - (IFS=','; for i in $(SRC_DIRS); do find $$i -type f -name '*.yaml' -exec rm -f {} \;; done) +.PHONY: clean-crds-capkk +clean-crds-capkk: ## Remove the generated crds for capkk + @rm -rf ./config/capkk/crds +#.PHONY: clean-release-git +#clean-release-git: ## Restores the git files usually modified during a release +# git restore ./*manager_image_patch.yaml ./*manager_pull_policy.yaml +# +#.PHONY: clean-generated-yaml +#clean-generated-yaml: ## Remove files generated by conversion-gen from the mentioned dirs. Example SRC_DIRS="./api/v1beta1" +# (IFS=','; for i in $(SRC_DIRS); do find $$i -type f -name '*.yaml' -exec rm -f {} \;; done) +# .PHONY: clean-generated-deepcopy clean-generated-deepcopy: ## Remove files generated by conversion-gen from the mentioned dirs. Example SRC_DIRS="./api/v1beta1" - (IFS=','; for i in $(SRC_DIRS); do find $$i -type f -name 'zz_generated.deepcopy*' -exec rm -f {} \;; done) + @(IFS=','; for i in $(SRC_DIRS); do find $$i -type f -name 'zz_generated.deepcopy*' -exec rm -f {} \;; done) ## -------------------------------------- ## Hack / Tools @@ -548,54 +328,32 @@ clean-generated-deepcopy: ## Remove files generated by conversion-gen from the m ##@ hack/tools: -.PHONY: $(CONTROLLER_GEN_BIN) -$(CONTROLLER_GEN_BIN): $(CONTROLLER_GEN) ## Build a local copy of controller-gen. - -.PHONY: $(GOTESTSUM_BIN) -$(GOTESTSUM_BIN): $(GOTESTSUM) ## Build a local copy of gotestsum. - -.PHONY: $(KUSTOMIZE_BIN) -$(KUSTOMIZE_BIN): $(KUSTOMIZE) ## Build a local copy of kustomize. - -.PHONY: $(SETUP_ENVTEST_BIN) -$(SETUP_ENVTEST_BIN): $(SETUP_ENVTEST) ## Build a local copy of setup-envtest. - -.PHONY: $(GOLANGCI_LINT_BIN) -$(GOLANGCI_LINT_BIN): $(GOLANGCI_LINT) ## Build a local copy of golangci-lint - -$(CONTROLLER_GEN): # Build controller-gen from tools folder. - GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(CONTROLLER_GEN_PKG) $(CONTROLLER_GEN_BIN) $(CONTROLLER_GEN_VER) - -$(GOTESTSUM): # Build gotestsum from tools folder. - GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(GOTESTSUM_PKG) $(GOTESTSUM_BIN) $(GOTESTSUM_VER) - -$(KUSTOMIZE): # Build kustomize from tools folder. - CGO_ENABLED=0 GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(KUSTOMIZE_PKG) $(KUSTOMIZE_BIN) $(KUSTOMIZE_VER) - -$(SETUP_ENVTEST): # Build setup-envtest from tools folder. - GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(SETUP_ENVTEST_PKG) $(SETUP_ENVTEST_BIN) $(SETUP_ENVTEST_VER) - -$(GOLANGCI_LINT): .github/workflows/golangci-lint.yml # Download golangci-lint using hack script into tools folder. - hack/ensure-golangci-lint.sh \ - -b $(TOOLS_BIN_DIR) \ - $(shell cat .github/workflows/golangci-lint.yml | grep [[:space:]]version | sed 's/.*version: //') - -# build the artifact of repository iso -ISO_ARCH ?= amd64 -ISO_OUTPUT_DIR ?= ./output -ISO_BUILD_WORKDIR := hack/gen-repository-iso -ISO_OS_NAMES := centos7 debian9 debian10 ubuntu1604 ubuntu1804 ubuntu2004 ubuntu2204 -ISO_BUILD_NAMES := $(addprefix build-iso-,$(ISO_OS_NAMES)) -build-iso-all: $(ISO_BUILD_NAMES) -.PHONY: $(ISO_BUILD_NAMES) -$(ISO_BUILD_NAMES): - @export DOCKER_BUILDKIT=1 - docker build \ - --platform linux/$(ISO_ARCH) \ - --build-arg TARGETARCH=$(ISO_ARCH) \ - -o type=local,dest=$(ISO_OUTPUT_DIR) \ - -f $(ISO_BUILD_WORKDIR)/dockerfile.$(subst build-iso-,,$@) \ - $(ISO_BUILD_WORKDIR) - -go-releaser-test: - goreleaser release --rm-dist --skip-publish --snapshot +$(CONTROLLER_GEN): # Build controller-gen into tools folder. + @if [ ! -f $(CONTROLLER_GEN) ]; then \ + CGO_ENABLED=0 GOBIN=$(OUTPUT_TOOLS_DIR) $(GO_INSTALL) $(CONTROLLER_GEN_PKG) $(CONTROLLER_GEN_BIN) $(CONTROLLER_GEN_VER); \ + fi + +$(GOTESTSUM): # Build gotestsum into tools folder. + @if [ ! -f $(GOTESTSUM) ]; then \ + CGO_ENABLED=0 GOBIN=$(OUTPUT_TOOLS_DIR) $(GO_INSTALL) $(GOTESTSUM_PKG) $(GOTESTSUM_BIN) $(GOTESTSUM_VER); \ + fi + +$(KUSTOMIZE): # Build kustomize into tools folder. + @if [ ! -f $(KUSTOMIZE) ]; then \ + CGO_ENABLED=0 GOBIN=$(OUTPUT_TOOLS_DIR) $(GO_INSTALL) $(KUSTOMIZE_PKG) $(KUSTOMIZE_BIN) $(KUSTOMIZE_VER); \ + fi + +$(SETUP_ENVTEST): # Build setup-envtest into tools folder. + if [ ! -f $(SETUP_ENVTEST) ]; then \ + CGO_ENABLED=0 GOBIN=$(OUTPUT_TOOLS_DIR) $(GO_INSTALL) $(SETUP_ENVTEST_PKG) $(SETUP_ENVTEST_BIN) $(SETUP_ENVTEST_VER); \ + fi + +$(GOLANGCI_LINT): # Build golangci-lint into tools folder. + @if [ ! -f $(GOLANGCI_LINT) ]; then \ + CGO_ENABLED=0 GOBIN=$(OUTPUT_TOOLS_DIR) $(GO_INSTALL) $(GOLANGCI_LINT_PKG) $(GOLANGCI_LINT_BIN) $(GOLANGCI_LINT_VER); \ + fi + +$(GORELEASER): # Build goreleaser into tools folder. + @if [ ! -f $(GORELEASER) ]; then \ + CGO_ENABLED=0 GOBIN=$(OUTPUT_TOOLS_DIR) $(GO_INSTALL) $(GORELEASER_PKG) $(GORELEASER_BIN) $(GORELEASER_VER); \ + fi diff --git a/OWNERS b/OWNERS index 6492db820..fcdea11d0 100644 --- a/OWNERS +++ b/OWNERS @@ -1,15 +1,13 @@ approvers: - pixiake - - 24sama - - rayzhou2017 - - littleBlackHouse + - liangzai006 + - redscholar + - zuoxuesong-worker reviewers: - pixiake - - Forest-L - rayzhou2017 - zryfish - - shaowenchen - benjaminhuo - calvinyv - FeynmanZhou @@ -17,4 +15,5 @@ reviewers: - wansir - LinuxSuRen - 24sama - - littleBlackHouse + - liangzai006 + - redscholar diff --git a/PROJECT b/PROJECT deleted file mode 100644 index d710397d1..000000000 --- a/PROJECT +++ /dev/null @@ -1,73 +0,0 @@ -domain: cluster.x-k8s.io -layout: -- go.kubebuilder.io/v3 -plugins: - manifests.sdk.operatorframework.io/v2: {} - scorecard.sdk.operatorframework.io/v2: {} -projectName: cluster-api-provider-kubekey -repo: github.com/kubesphere/kubekey -resources: -- api: - crdVersion: v1 - namespaced: true - controller: true - domain: cluster.x-k8s.io - group: infrastructure - kind: KKCluster - path: github.com/kubesphere/kubekey/api/v1beta1 - version: v1beta1 - webhooks: - defaulting: true - validation: true - webhookVersion: v1 -- api: - crdVersion: v1 - namespaced: true - domain: cluster.x-k8s.io - group: infrastructure - kind: KKClusterTemplate - path: github.com/kubesphere/kubekey/api/v1beta1 - version: v1beta1 - webhooks: - defaulting: true - validation: true - webhookVersion: v1 -- api: - crdVersion: v1 - namespaced: true - controller: true - domain: cluster.x-k8s.io - group: infrastructure - kind: KKMachine - path: github.com/kubesphere/kubekey/api/v1beta1 - version: v1beta1 - webhooks: - defaulting: true - validation: true - webhookVersion: v1 -- api: - crdVersion: v1 - namespaced: true - domain: cluster.x-k8s.io - group: infrastructure - kind: KKMachineTemplate - path: github.com/kubesphere/kubekey/api/v1beta1 - version: v1beta1 - webhooks: - defaulting: true - validation: true - webhookVersion: v1 -- api: - crdVersion: v1 - namespaced: true - controller: true - domain: cluster.x-k8s.io - group: infrastructure - kind: KKInstance - path: github.com/kubesphere/kubekey/api/v1beta1 - version: v1beta1 - webhooks: - defaulting: true - validation: true - webhookVersion: v1 -version: "3" diff --git a/README.md b/README.md index 3f058c8d7..3f06f2b29 100644 --- a/README.md +++ b/README.md @@ -1,408 +1,29 @@ -
- -[![CI](https://github.com/kubesphere/kubekey/workflows/CI/badge.svg?branch=master&event=push)](https://github.com/kubesphere/kubekey/actions?query=event%3Apush+branch%3Amaster+workflow%3ACI+) - -> English | [中文](README_zh-CN.md) - -### 👋 Welcome to KubeKey! - -KubeKey is an open-source lightweight tool for deploying Kubernetes clusters. It provides a flexible, rapid, and convenient way to install Kubernetes/K3s only, both Kubernetes/K3s and KubeSphere, and related cloud-native add-ons. It is also an efficient tool to scale and upgrade your cluster. - -In addition, KubeKey also supports customized Air-Gap package, which is convenient for users to quickly deploy clusters in offline environments. - -> KubeKey has passed [CNCF kubernetes conformance verification](https://www.cncf.io/certification/software-conformance/). - -Use KubeKey in the following three scenarios. - -* Install Kubernetes/K3s only -* Install Kubernetes/K3s and KubeSphere together in one command -* Install Kubernetes/K3s first, then deploy KubeSphere on it using [ks-installer](https://github.com/kubesphere/ks-installer) - -> **Important:** If you have existing Kubernetes clusters, please refer to [ks-installer (Install KubeSphere on existing Kubernetes cluster)](https://github.com/kubesphere/ks-installer). - -## Supported Environment - -### Linux Distributions - -* **Ubuntu** *16.04, 18.04, 20.04, 22.04* -* **Debian** *Bullseye, Buster, Stretch* -* **CentOS/RHEL** *7* -* **AlmaLinux** *9.0* -* **SUSE Linux Enterprise Server** *15* - -> Recommended Linux Kernel Version: `4.15 or later` -> You can run the `uname -srm` command to check the Linux Kernel Version. - -### Kubernetes Versions - -* **v1.19**:   *v1.19.15* -* **v1.20**:   *v1.20.10* -* **v1.21**:   *v1.21.14* -* **v1.22**:   *v1.22.15* -* **v1.23**:   *v1.23.10* (default) -* **v1.24**:   *v1.24.7* -* **v1.25**:   *v1.25.3* - -> Looking for more supported versions: \ -> [Kubernetes Versions](./docs/kubernetes-versions.md) \ -> [K3s Versions](./docs/k3s-versions.md) - -### Container Manager - -* **Docker** / **containerd** / **CRI-O** / **iSula** - -> `Kata Containers` can be set to automatically install and configure runtime class for it when the container manager is containerd or CRI-O. - -### Network Plugins - -* **Calico** / **Flannel** / **Cilium** / **Kube-OVN** / **Multus-CNI** - -> Kubekey also supports users to set the network plugin to `none` if there is a requirement for custom network plugin. - -## Requirements and Recommendations - -* Minimum resource requirements (For Minimal Installation of KubeSphere only): - * 2 vCPUs - * 4 GB RAM - * 20 GB Storage - -> /var/lib/docker is mainly used to store the container data, and will gradually increase in size during use and operation. In the case of a production environment, it is recommended that /var/lib/docker mounts a drive separately. - -* OS requirements: - * `SSH` can access to all nodes. - * Time synchronization for all nodes. - * `sudo`/`curl`/`openssl` should be used in all nodes. - * `docker` can be installed by yourself or by KubeKey. - * `Red Hat` includes `SELinux` in its `Linux release`. It is recommended to close SELinux or [switch the mode of SELinux](./docs/turn-off-SELinux.md) to `Permissive` - -> * It's recommended that Your OS is clean (without any other software installed), otherwise there may be conflicts. -> * A container image mirror (accelerator) is recommended to be prepared if you have trouble downloading images from dockerhub.io. [Configure registry-mirrors for the Docker daemon](https://docs.docker.com/registry/recipes/mirror/#configure-the-docker-daemon). -> * KubeKey will install [OpenEBS](https://openebs.io/) to provision LocalPV for development and testing environment by default, this is convenient for new users. For production, please use NFS / Ceph / GlusterFS or commercial products as persistent storage, and install the [relevant client](docs/storage-client.md) in all nodes. -> * If you encounter `Permission denied` when copying, it is recommended to check [SELinux and turn off it](./docs/turn-off-SELinux.md) first - -* Dependency requirements: - -KubeKey can install Kubernetes and KubeSphere together. Some dependencies need to be installed before installing kubernetes after version 1.18. You can refer to the list below to check and install the relevant dependencies on your node in advance. - -| | Kubernetes Version ≥ 1.18 | -| ------------- | -------------------------- | -| `socat` | Required | -| `conntrack` | Required | -| `ebtables` | Optional but recommended | -| `ipset` | Optional but recommended | -| `ipvsadm` | Optional but recommended | - -* Networking and DNS requirements: - * Make sure the DNS address in `/etc/resolv.conf` is available. Otherwise, it may cause some issues of DNS in cluster. - * If your network configuration uses Firewall or Security Group,you must ensure infrastructure components can communicate with each other through specific ports. It's recommended that you turn off the firewall or follow the link configuriation: [NetworkAccess](docs/network-access.md). - -## Usage - -### Get the KubeKey Executable File - -* The fastest way to get KubeKey is to use the script: - - ``` - curl -sfL https://get-kk.kubesphere.io | sh - - ``` -* Binary downloads of the KubeKey also can be found on the [Releases page](https://github.com/kubesphere/kubekey/releases). - Unpack the binary and you are good to go! -* Build Binary from Source Code - - ```shell - git clone https://github.com/kubesphere/kubekey.git - cd kubekey - make kk - ``` - -### Create a Cluster - -#### Quick Start - -Quick Start is for `all-in-one` installation which is a good start to get familiar with Kubernetes and KubeSphere. - -> Note: Since Kubernetes temporarily does not support uppercase NodeName, contains uppercase letters in the hostname will lead to subsequent installation error - -##### Command - -> If you have problem to access `https://storage.googleapis.com`, execute first `export KKZONE=cn`. +# 背景 +当前kubekey中,如果要添加命令,或修改命令,都需要提交代码并重新发版。扩展性较差。 +1. 任务与框架分离(优势,目的,更方便扩展,借鉴ansible的playbook设计) +2. 支持gitops(可通过git方式,管理自动化任务) +3. 支持connector扩展 +4. 支持云原生方式自动化批量任务管理 +# 安装kubekey +## kubernetes中安装 ```shell -./kk create cluster [--with-kubernetes version] [--with-kubesphere version] +helm upgrade --install --create-namespace -n kubekey-system kubekey kubekey-1.0.0.tgz ``` +然后通过创建 `Inventory` 和 `Playbook` 资源来执行命令 +**Inventory**: 任务执行的host清单. 用于定义与host相关, 与任务模板无关的变量. 详见[参数定义](docs/zh/201-variable.md) +**Playbook**: playbook的配置信息,在哪些host中执行,执行哪个playbook文件, 执行时参数等等。 -##### Examples - -* Create a pure Kubernetes cluster with default version (Kubernetes v1.23.10). - - ```shell - ./kk create cluster - ``` -* Create a Kubernetes cluster with a specified version. - - ```shell - ./kk create cluster --with-kubernetes v1.24.1 --container-manager containerd - ``` -* Create a Kubernetes cluster with KubeSphere installed. - - ```shell - ./kk create cluster --with-kubesphere v3.2.1 - ``` - -#### Advanced - -You have more control to customize parameters or create a multi-node cluster using the advanced installation. Specifically, create a cluster by specifying a configuration file. - -> If you have problem to access `https://storage.googleapis.com`, execute first `export KKZONE=cn`. - -1. First, create an example configuration file - - ```shell - ./kk create config [--with-kubernetes version] [--with-kubesphere version] [(-f | --filename) path] - ``` - - **examples:** - - * create an example config file with default configurations. You also can specify the file that could be a different filename, or in different folder. - - ```shell - ./kk create config [-f ~/myfolder/abc.yaml] - ``` - - * with KubeSphere - - ```shell - ./kk create config --with-kubesphere v3.2.1 - ``` -2. Modify the file config-sample.yaml according to your environment - -> Note: Since Kubernetes temporarily does not support uppercase NodeName, contains uppercase letters in workerNode`s name will lead to subsequent installation error -> -> A persistent storage is required in the cluster, when kubesphere will be installed. The local volume is used default. If you want to use other persistent storage, please refer to [addons](./docs/addons.md). - -3. Create a cluster using the configuration file - - ```shell - ./kk create cluster -f config-sample.yaml - ``` - -### Enable Multi-cluster Management - -By default, KubeKey will only install a **solo** cluster without Kubernetes federation. If you want to set up a multi-cluster control plane to centrally manage multiple clusters using KubeSphere, you need to set the `ClusterRole` in [config-example.yaml](docs/config-example.md). For multi-cluster user guide, please refer to [How to Enable the Multi-cluster Feature](https://github.com/kubesphere/community/tree/master/sig-multicluster/how-to-setup-multicluster-on-kubesphere). - -### Enable Pluggable Components - -KubeSphere has decoupled some core feature components since v2.1.0. These components are designed to be pluggable which means you can enable them either before or after installation. By default, KubeSphere will be started with a minimal installation if you do not enable them. - -You can enable any of them according to your demands. It is highly recommended that you install these pluggable components to discover the full-stack features and capabilities provided by KubeSphere. Please ensure your machines have sufficient CPU and memory before enabling them. See [Enable Pluggable Components](https://github.com/kubesphere/ks-installer#enable-pluggable-components) for the details. - -### Add Nodes - -Add new node's information to the cluster config file, then apply the changes. - +## 二进制执行 +可直接用二进制在命令行中执行命令 ```shell -./kk add nodes -f config-sample.yaml +kk run -i inventory.yaml -c config.yaml playbook.yaml ``` +运行命令后, 会在工作目录的runtime下生成对应的 `Inventory` 和 `Playbook` 资源 -### Delete Nodes - -You can delete the node by the following command,the nodeName that needs to be removed. - -```shell -./kk delete node -f config-sample.yaml -``` - -### Delete Cluster - -You can delete the cluster by the following command: - -* If you started with the quick start (all-in-one): - -```shell -./kk delete cluster -``` - -* If you started with the advanced (created with a configuration file): - -```shell -./kk delete cluster [-f config-sample.yaml] -``` - -### Upgrade Cluster - -#### Allinone - -Upgrading cluster with a specified version. - -```shell -./kk upgrade [--with-kubernetes version] [--with-kubesphere version] -``` - -* Support upgrading Kubernetes only. -* Support upgrading KubeSphere only. -* Support upgrading Kubernetes and KubeSphere. - -#### Multi-nodes - -Upgrading cluster with a specified configuration file. - -```shell -./kk upgrade [--with-kubernetes version] [--with-kubesphere version] [(-f | --filename) path] -``` - -* If `--with-kubernetes` or `--with-kubesphere` is specified, the configuration file will be also updated. -* Use `-f` to specify the configuration file which was generated for cluster creation. - -> Note: Upgrading multi-nodes cluster need a specified configuration file. If the cluster was installed without kubekey or the configuration file for installation was not found, the configuration file needs to be created by yourself or following command. - -Getting cluster info and generating kubekey's configuration file (optional). - -```shell -./kk create config [--from-cluster] [(-f | --filename) path] [--kubeconfig path] -``` - -* `--from-cluster` means fetching cluster's information from an existing cluster. -* `-f` refers to the path where the configuration file is generated. -* `--kubeconfig` refers to the path where the kubeconfig. -* After generating the configuration file, some parameters need to be filled in, such as the ssh information of the nodes. - -## Documents - -* [Features List](docs/features.md) -* [Commands](docs/commands/kk.md) -* [Configuration example](docs/config-example.md) -* [Air-Gapped Installation](docs/manifest_and_artifact.md) -* [Highly Available clusters](docs/ha-mode.md) -* [Addons](docs/addons.md) -* [Network access](docs/network-access.md) -* [Storage clients](docs/storage-client.md) -* [kubectl auto-completion](docs/kubectl-autocompletion.md) -* [kubekey auto-completion](docs/kubekey-autocompletion.md) -* [Roadmap](docs/roadmap.md) -* [Check-Renew-Certificate](docs/check-renew-certificate.md) -* [Developer-Guide](docs/developer-guide.md) - -## Contributors ✨ - -Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)): - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
pixiake
pixiake

💻 📖
Forest
Forest

💻 📖
rayzhou2017
rayzhou2017

💻 📖
shaowenchen
shaowenchen

💻 📖
Zhao Xiaojie
Zhao Xiaojie

💻 📖
Zack Zhang
Zack Zhang

💻
Akhil Mohan
Akhil Mohan

💻
pengfei
pengfei

📖
min zhang
min zhang

💻 📖
zgldh
zgldh

💻
xrjk
xrjk

💻
yonghongshi
yonghongshi

💻
Honglei
Honglei

📖
liucy1983
liucy1983

💻
Lien
Lien

📖
Tony Wang
Tony Wang

📖
Hongliang Wang
Hongliang Wang

💻
dawn
dawn

💻
Duan Jiong
Duan Jiong

💻
calvinyv
calvinyv

📖
Benjamin Huo
Benjamin Huo

📖
Sherlock113
Sherlock113

📖
fu_changjie
fu_changjie

📖
yuswift
yuswift

💻
ruiyaoOps
ruiyaoOps

📖
LXM
LXM

📖
sbhnet
sbhnet

💻
misteruly
misteruly

💻
John Niang
John Niang

📖
Michael Li
Michael Li

💻
独孤昊天
独孤昊天

💻
Liu Shaohui
Liu Shaohui

💻
Leo Li
Leo Li

💻
Roland
Roland

💻
Vinson Zou
Vinson Zou

📖
tag_gee_y
tag_gee_y

💻
codebee
codebee

💻
Daniel Owen van Dommelen
Daniel Owen van Dommelen

🤔
Naidile P N
Naidile P N

💻
Haiker Sun
Haiker Sun

💻
Jing Yu
Jing Yu

💻
Chauncey
Chauncey

💻
Tan Guofu
Tan Guofu

💻
lvillis
lvillis

📖
Vincent He
Vincent He

💻
laminar
laminar

💻
tongjin
tongjin

💻
Reimu
Reimu

💻
Ikko Ashimine
Ikko Ashimine

📖
Ben Ye
Ben Ye

💻
yinheli
yinheli

💻
hellocn9
hellocn9

💻
Brandan Schmitz
Brandan Schmitz

💻
yjqg6666
yjqg6666

📖 💻
失眠是真滴难受
失眠是真滴难受

💻
mango
mango

👀
wenwutang
wenwutang

💻
Shiny Hou
Shiny Hou

💻
zhouqiu0103
zhouqiu0103

💻
77yu77
77yu77

💻
hzhhong
hzhhong

💻
zhang-wei
zhang-wei

💻
Deshi Xiao
Deshi Xiao

💻 📖
besscroft
besscroft

📖
张志强
张志强

💻
lwabish
lwabish

💻 📖
qyz87
qyz87

💻
ZhengJin Fang
ZhengJin Fang

💻
Eric_Lian
Eric_Lian

💻
nicognaw
nicognaw

💻
吕德庆
吕德庆

💻
littleplus
littleplus

💻
Konstantin
Konstantin

🤔
kiragoo
kiragoo

💻
jojotong
jojotong

💻
littleBlackHouse
littleBlackHouse

💻 📖
guangwu
guangwu

💻 📖
wongearl
wongearl

💻
wenwenxiong
wenwenxiong

💻
柏喵Sakura
柏喵Sakura

💻
cui fliter
cui fliter

📖
刘旭
刘旭

💻
- - - - - +# 文档 +**[项目模版编写规范](docs/zh/001-project.md)** +**[模板语法](docs/zh/101-syntax.md)** +**[参数定义](docs/zh/201-variable.md)** +**[集群管理](docs/zh/core/README.md)** -This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome! diff --git a/README_zh-CN.md b/README_zh-CN.md deleted file mode 100644 index 753920d7a..000000000 --- a/README_zh-CN.md +++ /dev/null @@ -1,423 +0,0 @@ -# KubeKey - -[![CI](https://github.com/kubesphere/kubekey/workflows/CI/badge.svg?branch=master&event=push)](https://github.com/kubesphere/kubekey/actions?query=event%3Apush+branch%3Amaster+workflow%3ACI+) - -> [English](README.md) | 中文 - -KubeKey是一个开源的轻量级工具,用于部署Kubernetes集群。它提供了一种灵活、快速、方便的方式来安装Kubernetes/K3s、Kubernetes/K3s和KubeSphere,以及相关的云原生附加组件。它也是扩展和升级集群的有效工具。 - -此外,KubeKey还支持定制离线包(artifact),方便用户在离线环境下快速部署集群。 - -> KubeKey 通过了 [CNCF kubernetes 一致性认证](https://www.cncf.io/certification/software-conformance/)。 - -有三种情况可以使用 KubeKey。 - -* 仅安装 Kubernetes -* 用一个命令中安装 Kubernetes 和 KubeSphere -* 首先安装 Kubernetes,然后使用 [ks-installer](https://github.com/kubesphere/ks-installer) 在其上部署 KubeSphere - -> 重要提示:Kubekey 将会帮您安装 Kubernetes,若已有 Kubernetes 集群请参考 [在 Kubernetes 之上安装 KubeSphere](https://github.com/kubesphere/ks-installer/)。 - -## 优势 - -* 基于 Ansible 的安装程序具有大量软件依赖性,例如 Python。KubeKey 是使用 Go 语言开发的,可以消除在各种环境中出现的问题,从而提高安装成功率。 -* KubeKey 使用 Kubeadm 在节点上尽可能多地并行安装 K8s 集群,以降低安装复杂性并提高效率。与较早的安装程序相比,它将大大节省安装时间。 -* KubeKey 支持将集群从 all-in-one 扩展到多节点集群甚至 HA 集群。 -* KubeKey 旨在将集群当作一个对象操作,即 CaaO。 - -## 支持的环境 - -### Linux 发行版 - -* **Ubuntu** *16.04, 18.04, 20.04, 22.04* -* **Debian** *Bullseye, Buster, Stretch* -* **CentOS/RHEL** *7* -* **AlmaLinux** *9.0* -* **SUSE Linux Enterprise Server** *15* - -> 建议使用 Linux Kernel 版本: `4.15 or later` -> 可以通过命令 `uname -srm` 查看 Linux Kernel 版本。 - -### Kubernetes 版本 - -* **v1.19**:   *v1.19.15* -* **v1.20**:   *v1.20.10* -* **v1.21**:   *v1.21.14* -* **v1.22**:   *v1.22.15* -* **v1.23**:   *v1.23.10* (default) -* **v1.24**:   *v1.24.7* -* **v1.25**:   *v1.25.3* - -> 查看更多支持的版本: \ -> [Kubernetes 版本](./docs/kubernetes-versions.md) \ -> [K3s 版本](./docs/k3s-versions.md) - -## 要求和建议 - -* 最低资源要求(仅对于最小安装 KubeSphere): - * 2 核虚拟 CPU - * 4 GB 内存 - * 20 GB 储存空间 - -> /var/lib/docker 主要用于存储容器数据,在使用和操作过程中会逐渐增大。对于生产环境,建议 /var/lib/docker 单独挂盘。 - -* 操作系统要求: - * `SSH` 可以访问所有节点。 - * 所有节点的时间同步。 - * `sudo`/`curl`/`openssl` 应在所有节点使用。 - * `docker` 可以自己安装,也可以通过 KubeKey 安装。 - * `Red Hat` 在其 `Linux` 发行版本中包括了 `SELinux`,建议[关闭SELinux](./docs/turn-off-SELinux_zh-CN.md)或者将[SELinux的模式切换](./docs/turn-off-SELinux_zh-CN.md)为Permissive[宽容]工作模式 - -> * 建议您的操作系统环境足够干净 (不安装任何其他软件),否则可能会发生冲突。 -> * 如果在从 dockerhub.io 下载镜像时遇到问题,建议准备一个容器镜像仓库 (加速器)。[为 Docker 守护程序配置镜像加速](https://docs.docker.com/registry/recipes/mirror/#configure-the-docker-daemon)。 -> * 默认情况下,KubeKey 将安装 [OpenEBS](https://openebs.io/) 来为开发和测试环境配置 LocalPV,这对新用户来说非常方便。对于生产,请使用 NFS/Ceph/GlusterFS 或商业化存储作为持久化存储,并在所有节点中安装[相关的客户端](./docs/storage-client.md) 。 -> * 如果遇到拷贝时报权限问题Permission denied,建议优先考虑查看[SELinux的原因](./docs/turn-off-SELinux_zh-CN.md)。 - -* 依赖要求: - -KubeKey 可以同时安装 Kubernetes 和 KubeSphere。在版本1.18之后,安装kubernetes前需要安装一些依赖。你可以参考下面的列表,提前在你的节点上检查并安装相关依赖。 - -| | Kubernetes 版本 ≥ 1.18 | -| ------------- | ----------------------- | -| `socat` | 必须安装 | -| `conntrack` | 必须安装 | -| `ebtables` | 可选,但推荐安装 | -| `ipset` | 可选,但推荐安装 | -| `ipvsadm` | 可选,但推荐安装 | - -* 网络和 DNS 要求: - * 确保 `/etc/resolv.conf` 中的 DNS 地址可用。否则,可能会导致集群中出现某些 DNS 问题。 - * 如果您的网络配置使用防火墙或安全组,则必须确保基础结构组件可以通过特定端口相互通信。建议您关闭防火墙或遵循链接配置:[网络访问](docs/network-access.md)。 - -## 用法 - -### 获取安装程序可执行文件 - -* 使用脚本获取 KubeKey - > 如果无法访问 https://github.com, 请先执行 export KKZONE=cn. - ``` - curl -sfL https://get-kk.kubesphere.io | sh - - ``` - -* 下载KubeKey可执行文件 [Releases page](https://github.com/kubesphere/kubekey/releases) - - 下载解压后可直接使用。 - -* 从源代码生成二进制文件 - - ```shell - git clone https://github.com/kubesphere/kubekey.git - cd kubekey - make kk - ``` - -### 创建集群 - -#### 快速开始 - -快速入门使用 `all-in-one` 安装,这是熟悉 KubeSphere 的良好开始。 - -> 注意: 由于 Kubernetes 暂不支持大写 NodeName, hostname 中包含大写字母将导致后续安装过程无法正常结束 - -##### 命令 - -> 如果无法访问 `https://storage.googleapis.com`, 请先执行 `export KKZONE=cn`. - -```shell -./kk create cluster [--with-kubernetes version] [--with-kubesphere version] -``` - -##### 例子 - -* 使用默认版本创建一个纯 Kubernetes 集群 - - ```shell - ./kk create cluster - ``` -* 创建指定一个([支持的版本](#KubernetesVersions))的 Kubernetes 集群 - - ```shell - ./kk create cluster --with-kubernetes v1.19.8 - ``` -* 创建一个部署了 KubeSphere 的 Kubernetes 集群 (例如 `--with-kubesphere v3.1.0`) - - ```shell - ./kk create cluster --with-kubesphere [version] - ``` -* 创建一个指定的 container runtime 的 Kubernetes 集群(docker, crio, containerd and isula) - - ```shell - ./kk create cluster --container-manager containerd - ``` - -#### 高级用法 - -您可以使用高级安装来控制自定义参数或创建多节点集群。具体来说,通过指定配置文件来创建集群。 - -> 如果无法访问 `https://storage.googleapis.com`, 请先执行 `export KKZONE=cn`. - -1. 首先,创建一个示例配置文件 - - ```shell - ./kk create config [--with-kubernetes version] [--with-kubesphere version] [(-f | --filename) path] - ``` - - **例子:** - - * 使用默认配置创建一个示例配置文件。您也可以指定文件名称或文件所在的文件夹。 - - ```shell - ./kk create config [-f ~/myfolder/config-sample.yaml] - ``` - * 同时安装 KubeSphere - - ```shell - ./kk create config --with-kubesphere - ``` -2. 根据您的环境修改配置文件 config-sample.yaml - -> 注意: 由于 Kubernetes 暂不支持大写 NodeName, worker 节点名中包含大写字母将导致后续安装过程无法正常结束 -> -> 当指定安装KubeSphere时,要求集群中有可用的持久化存储。默认使用localVolume,如果需要使用其他持久化存储,请参阅 [addons](./docs/addons.md) 配置。 - -3. 使用配置文件创建集群。 - - ```shell - ./kk create cluster -f ~/myfolder/config-sample.yaml - ``` - -### 启用多集群管理 - -默认情况下,Kubekey 将仅安装一个 Solo 模式的单集群,即未开启 Kubernetes 多集群联邦。如果您希望将 KubeSphere 作为一个支持多集群集中管理的中央面板,您需要在 [config-example.yaml](docs/config-example.md) 中设置 `ClusterRole`。关于多集群的使用文档,请参考 [如何启用多集群](https://github.com/kubesphere/community/blob/master/sig-multicluster/how-to-setup-multicluster-on-kubesphere/README_zh.md)。 - -### 开启可插拔功能组件 - -KubeSphere 从 2.1.0 版本开始对 Installer 的各功能组件进行了解耦,快速安装将默认仅开启最小化安装(Minimal Installation),Installer 支持在安装前或安装后自定义可插拔的功能组件的安装。使最小化安装更快速轻量且资源占用更少,也方便不同用户按需选择安装不同的功能组件。 - -KubeSphere 有多个可插拔功能组件,功能组件的介绍可参考 [配置示例](docs/config-example.md)。您可以根据需求,选择开启安装 KubeSphere 的可插拔功能组件。我们非常建议您开启这些功能组件来体验 KubeSphere 完整的功能以及端到端的解决方案。请在安装前确保您的机器有足够的 CPU 与内存资源。开启可插拔功能组件可参考 [开启可选功能组件](https://github.com/kubesphere/ks-installer/blob/master/README_zh.md#%E5%AE%89%E8%A3%85%E5%8A%9F%E8%83%BD%E7%BB%84%E4%BB%B6)。 - -### 添加节点 - -将新节点的信息添加到集群配置文件,然后应用更改。 - -```shell -./kk add nodes -f config-sample.yaml -``` - -### 删除节点 - -通过以下命令删除节点,nodename指需要删除的节点名。 - -```shell -./kk delete node -f config-sample.yaml -``` - -### 删除集群 - -您可以通过以下命令删除集群: - -* 如果您以快速入门(all-in-one)开始: - -```shell -./kk delete cluster -``` - -* 如果从高级安装开始(使用配置文件创建的集群): - -```shell -./kk delete cluster [-f config-sample.yaml] -``` - -### 集群升级 - -#### 单节点集群 - -升级集群到指定版本。 - -```shell -./kk upgrade [--with-kubernetes version] [--with-kubesphere version] -``` - -* `--with-kubernetes` 指定kubernetes目标版本。 -* `--with-kubesphere` 指定kubesphere目标版本。 - -#### 多节点集群 - -通过指定配置文件对集群进行升级。 - -```shell -./kk upgrade [--with-kubernetes version] [--with-kubesphere version] [(-f | --filename) path] -``` - -* `--with-kubernetes` 指定kubernetes目标版本。 -* `--with-kubesphere` 指定kubesphere目标版本。 -* `-f` 指定集群安装时创建的配置文件。 - -> 注意: 升级多节点集群需要指定配置文件. 如果集群非kubekey创建,或者创建集群时生成的配置文件丢失,需要重新生成配置文件,或使用以下方法生成。 - -Getting cluster info and generating kubekey's configuration file (optional). - -```shell -./kk create config [--from-cluster] [(-f | --filename) path] [--kubeconfig path] -``` - -* `--from-cluster` 根据已存在集群信息生成配置文件. -* `-f` 指定生成配置文件路径. -* `--kubeconfig` 指定集群kubeconfig文件. -* 由于无法全面获取集群配置,生成配置文件后,请根据集群实际信息补全配置文件。 - -### 启用 kubectl 自动补全 - -KubeKey 不会启用 kubectl 自动补全功能。请参阅下面的指南并将其打开: - -**先决条件**:确保已安装 `bash-autocompletion` 并可以正常工作。 - -```shell -# 安装 bash-completion -apt-get install bash-completion - -# 将 completion 脚本添加到你的 ~/.bashrc 文件 -echo 'source <(kubectl completion bash)' >>~/.bashrc - -# 将 completion 脚本添加到 /etc/bash_completion.d 目录 -kubectl completion bash >/etc/bash_completion.d/kubectl -``` - -更详细的参考可以在[这里](https://kubernetes.io/docs/tasks/tools/install-kubectl/#enabling-shell-autocompletion)找到。 - -## 相关文档 - -* [特性列表](docs/features.md) -* [命令手册](docs/commands/kk.md) -* [配置示例](docs/config-example.md) -* [离线安装](docs/zh/manifest_and_artifact.md) -* [高可用集群](docs/ha-mode.md) -* [自定义插件安装](docs/addons.md) -* [网络访问](docs/network-access.md) -* [存储客户端](docs/storage-client.md) -* [路线图](docs/roadmap.md) -* [查看或更新证书](docs/check-renew-certificate.md) -* [开发指南](docs/developer-guide.md) - -## 贡献者 ✨ - -欢迎任何形式的贡献! 感谢这些优秀的贡献者,是他们让我们的项目快速成长。 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
pixiake
pixiake

💻 📖
Forest
Forest

💻 📖
rayzhou2017
rayzhou2017

💻 📖
shaowenchen
shaowenchen

💻 📖
Zhao Xiaojie
Zhao Xiaojie

💻 📖
Zack Zhang
Zack Zhang

💻
Akhil Mohan
Akhil Mohan

💻
pengfei
pengfei

📖
min zhang
min zhang

💻 📖
zgldh
zgldh

💻
xrjk
xrjk

💻
yonghongshi
yonghongshi

💻
Honglei
Honglei

📖
liucy1983
liucy1983

💻
Lien
Lien

📖
Tony Wang
Tony Wang

📖
Hongliang Wang
Hongliang Wang

💻
dawn
dawn

💻
Duan Jiong
Duan Jiong

💻
calvinyv
calvinyv

📖
Benjamin Huo
Benjamin Huo

📖
Sherlock113
Sherlock113

📖
fu_changjie
fu_changjie

📖
yuswift
yuswift

💻
ruiyaoOps
ruiyaoOps

📖
LXM
LXM

📖
sbhnet
sbhnet

💻
misteruly
misteruly

💻
John Niang
John Niang

📖
Michael Li
Michael Li

💻
独孤昊天
独孤昊天

💻
Liu Shaohui
Liu Shaohui

💻
Leo Li
Leo Li

💻
Roland
Roland

💻
Vinson Zou
Vinson Zou

📖
tag_gee_y
tag_gee_y

💻
codebee
codebee

💻
Daniel Owen van Dommelen
Daniel Owen van Dommelen

🤔
Naidile P N
Naidile P N

💻
Haiker Sun
Haiker Sun

💻
Jing Yu
Jing Yu

💻
Chauncey
Chauncey

💻
Tan Guofu
Tan Guofu

💻
lvillis
lvillis

📖
Vincent He
Vincent He

💻
laminar
laminar

💻
tongjin
tongjin

💻
Reimu
Reimu

💻
Ikko Ashimine
Ikko Ashimine

📖
Ben Ye
Ben Ye

💻
yinheli
yinheli

💻
hellocn9
hellocn9

💻
Brandan Schmitz
Brandan Schmitz

💻
yjqg6666
yjqg6666

📖 💻
失眠是真滴难受
失眠是真滴难受

💻
mango
mango

👀
wenwutang
wenwutang

💻
Shiny Hou
Shiny Hou

💻
zhouqiu0103
zhouqiu0103

💻
77yu77
77yu77

💻
hzhhong
hzhhong

💻
zhang-wei
zhang-wei

💻
Deshi Xiao
Deshi Xiao

💻 📖
besscroft
besscroft

📖
张志强
张志强

💻
lwabish
lwabish

💻 📖
qyz87
qyz87

💻
ZhengJin Fang
ZhengJin Fang

💻
Eric_Lian
Eric_Lian

💻
nicognaw
nicognaw

💻
吕德庆
吕德庆

💻
littleplus
littleplus

💻
Konstantin
Konstantin

🤔
kiragoo
kiragoo

💻
jojotong
jojotong

💻
littleBlackHouse
littleBlackHouse

💻 📖
guangwu
guangwu

💻 📖
wongearl
wongearl

💻
wenwenxiong
wenwenxiong

💻
柏喵Sakura
柏喵Sakura

💻
cui fliter
cui fliter

📖
刘旭
刘旭

💻
- - - - - diff --git a/api/capkk/infrastructure/v1beta1/kkcluster_types.go b/api/capkk/infrastructure/v1beta1/kkcluster_types.go new file mode 100644 index 000000000..964634e00 --- /dev/null +++ b/api/capkk/infrastructure/v1beta1/kkcluster_types.go @@ -0,0 +1,183 @@ +/* +Copyright 2024 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clusterv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +const ( + // KKClusterFinalizer allows ReconcileKKCluster to clean up KK resources associated with KKCluster before + // removing it from the API server. + KKClusterFinalizer = "kkcluster.infrastructure.cluster.x-k8s.io" + + // KKClusterSSHPrivateKeyAnnotation is the annotation for the secret used for SSH authentication. It contains the SSH private key + // and will be mounted in the executor pod. + KKClusterSSHPrivateKeyAnnotation = "kkcluster.infrastructure.cluster.x-k8s.io/ssh-auth" +) + +const ( + // KKClusterNodeReachedCondition represents the condition type indicating whether the hosts + // defined in the inventory are reachable. + KKClusterNodeReachedCondition clusterv1beta1.ConditionType = "NodeReached" + // KKClusterNodeReachedConditionReasonWaiting indicates that the node reachability check is pending. + // This check is triggered when the corresponding inventory host's configuration changes. + KKClusterNodeReachedConditionReasonWaiting = "waiting for node status check" + // KKClusterNodeReachedConditionReasonUnreached indicates that the node reachability check has failed. + // This means the node is currently offline or inaccessible. + KKClusterNodeReachedConditionReasonUnreached = "node is unreachable" + + // KKClusterKKMachineConditionReady represents the condition type indicating whether the associated inventory + // has been successfully marked as ready. + KKClusterKKMachineConditionReady clusterv1beta1.ConditionType = "KKClusterMachineReady" + // KKClusterKKMachineConditionReadyReasonWaiting indicates that the associated inventory is still being synchronized. + KKClusterKKMachineConditionReadyReasonWaiting = "waiting for kkmachine sync" + // KKMachineKKMachineConditionReasonSyncing indicates that the associated inventory has been successfully synchronized. + KKMachineKKMachineConditionReasonSyncing = "syncing for kkmachine" + // KKMachineKKMachineConditionReasonFailed indicates that the associated inventory synchronization process has failed. + KKMachineKKMachineConditionReasonFailed = "kkmachine run failed" +) + +type KKClusterFailedReason string + +const ( + // KKClusterFailedReasonUnknown like cannot get resource from kubernetes. + KKClusterFailedUnknown KKClusterFailedReason = "unknown" + // KKClusterFailedReasonInvalidHosts like hosts defined in kkcluster is invalid. + KKClusterFailedInvalidHosts KKClusterFailedReason = "hosts defined in kkcluster is invalid." + // KKClusterFailedReasonSyncInventory like failed to sync inventory. + KKClusterFailedSyncInventory KKClusterFailedReason = "failed to sync inventory" + // KKClusterFailedReasonSyncCPKKMachine like failed to sync control_plane kkmachine. + KKClusterFailedSyncCPKKMachine KKClusterFailedReason = "sync control_plane kkmachine failed." + // KKClusterFailedReasonSyncWorkerKKMachine like failed to sync worker kkmachine. + KKClusterFailedSyncWorkerKKMachine KKClusterFailedReason = "sync worker kkmachine failed." +) + +// ControlPlaneEndpointType defines the type of control plane endpoint used for communication with the cluster. +type ControlPlaneEndpointType string + +const ( + // ControlPlaneEndpointTypeDNS indicates the control plane endpoint is a globally resolvable DNS entry. + // ensuring that the configuration always points to the control plane nodes. + ControlPlaneEndpointTypeDNS ControlPlaneEndpointType = "dns" + // ControlPlaneEndpointTypeVIP(DEFAULT) indicates the control plane endpoint is a Virtual IP (VIP). + // - ARP Mode: Requires the management cluster and worker cluster nodes to be in the same network segment. + // - BGP Mode: Requires a network environment that supports BGP, with proper configuration in both + // the management and worker clusters. + ControlPlaneEndpointTypeVIP ControlPlaneEndpointType = "kube_vip" +) + +type InventoryHostConnector struct { + // Type to connector the host. + Type string `json:"type,omitempty"` + // Host address. default use host.name. + Host string `json:"host,omitempty"` + // User is the user name of the host. default is root. + // +optional + User string `json:"user,omitempty"` + // Password is the password of the host. + // +optional + Password string `json:"password,omitempty"` + // PrivateKey is the private key of the host. default is ~/.ssh/id_rsa. + // +optional + PrivateKey string `json:"privateKey,omitempty"` +} +type InventoryHost struct { + // Name of the host. + Name string `json:"name,omitempty"` + // Connector to connect the host. + Connector InventoryHostConnector `json:"connector,omitempty"` + // Vars for the host. + // +optional + // +kubebuilder:pruning:PreserveUnknownFields + Vars runtime.RawExtension `json:"vars,omitempty"` +} + +// KKClusterSpec defines the desired state of KKCluster. +type KKClusterSpec struct { + // InventoryHosts contains all hosts of the cluster. + InventoryHosts []InventoryHost `json:"inventory,omitempty"` + // which Group defined in Inventory will be checked. there is some default group by system: + // - all: contains all hosts + // - ungrouped: contains hosts which do not belong to any groups. + // if the value is empty, "ungrouped" will be used. + HostCheckGroup string `json:"hostCheckGroup,omitempty"` + // tolerate defines if tolerate host check if failed. + Tolerate bool `json:"tolerate,omitempty"` + // ControlPlaneEndpointType defines the type of control plane endpoint. such as dns, vip. + // when use vip, it will deploy kube-vip in each control_plane node. the default value is vip. + ControlPlaneEndpointType ControlPlaneEndpointType `json:"controlPlaneEndpointType,omitempty"` +} + +// KKClusterStatus defines the observed state of KKCluster. +type KKClusterStatus struct { + // if Ready to create cluster. usage after inventory is ready. + Ready bool `json:"ready,omitempty"` + + // FailureReason + FailureReason KKClusterFailedReason `json:"failureReason,omitempty"` + + FailureMessage string `json:"failureMessage,omitempty"` + + // Conditions defines current service state of the KKCluster. + // +optional + Conditions clusterv1beta1.Conditions `json:"conditions,omitempty"` +} + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +k8s:openapi-gen=true +// +kubebuilder:resource:scope=Namespaced,categories=cluster-api,shortName=kkc +// +kubebuilder:subresource:status +// +kubebuilder:metadata:labels="cluster.x-k8s.io/v1beta1=v1beta1" +// +kubebuilder:printcolumn:name="Cluster",type="string",JSONPath=".metadata.labels.cluster\\.x-k8s\\.io/cluster-name",description="Cluster to which this KKClusters belongs" +// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.ready",description="Cluster infrastructure is ready for SSH instances" +// +kubebuilder:printcolumn:name="ControlPlaneEndpointType",type="string",JSONPath=".spec.controlPlaneEndpointType",description="the ControlPlaneEndpointType to connect workload cluster" + +// KKCluster resource maps a kubernetes cluster, manage and reconcile cluster status. +type KKCluster struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec KKClusterSpec `json:"spec,omitempty"` + Status KKClusterStatus `json:"status,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// KKClusterList of KKCluster +type KKClusterList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []KKCluster `json:"items"` +} + +func init() { + SchemeBuilder.Register(&KKCluster{}, &KKClusterList{}) +} + +// GetConditions returns the observations of the operational state of the KKCluster resource. +func (k *KKCluster) GetConditions() clusterv1beta1.Conditions { + return k.Status.Conditions +} + +// SetConditions sets the underlying service state of the KKCluster to the predescribed clusterv1beta1.Conditions. +func (k *KKCluster) SetConditions(conditions clusterv1beta1.Conditions) { + k.Status.Conditions = conditions +} diff --git a/api/capkk/infrastructure/v1beta1/kkmachine_types.go b/api/capkk/infrastructure/v1beta1/kkmachine_types.go new file mode 100644 index 000000000..da8f81c0f --- /dev/null +++ b/api/capkk/infrastructure/v1beta1/kkmachine_types.go @@ -0,0 +1,139 @@ +/* +Copyright 2024 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clusterv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +const ( + // MachineFinalizer allows ReconcileKKMachine to clean up KubeKey resources associated with KKMachine before + // removing it from the apiserver. + KKMachineFinalizer = "kkmachine.infrastructure.cluster.x-k8s.io" + + // KKMachineBelongGroupLabel defines which kkmachine belong to. + KKMachineBelongGroupLabel = "kkmachine.infrastructure.cluster.x-k8s.io/group" + + // AddNodePlaybookAnnotation add node to cluster. + AddNodePlaybookAnnotation = "playbook.kubekey.kubesphere.io/add-node" + // DeleteNodePlaybookAnnotation remove node from cluster. + DeleteNodePlaybookAnnotation = "playbook.kubekey.kubesphere.io/delete-node" +) + +type KKMachineFailedReason string + +const ( + // KKMachineFailedReasonAddNodeFailed add node failed. + KKMachineFailedReasonAddNodeFailed KKMachineFailedReason = "add node failed" + // KKMachineFailedReasonDeleteNodeFailed delete node failed. + KKMachineFailedReasonDeleteNodeFailed clusterv1beta1.ConditionType = "delete failed failed" +) + +// KKMachineSpec defines the desired state of KKMachine. +type KKMachineSpec struct { + // Roles defines the roles assigned to the Kubernetes cluster node, such as "worker" or "control-plane". + // A KKMachine created by ControlPlane will automatically have the "control-plane" role. + // A KKMachine created by MachineDeployment will automatically have the "worker" role. + // Additional custom roles can also be specified in this field as needed. + Roles []string `json:"roles,omitempty"` + + // providerID is the identification ID of the machine provided by the provider. + // This field must match the provider ID as seen on the node object corresponding to this machine. + // This field is required by higher level consumers of cluster-api. Example use case is cluster autoscaler + // with cluster-api as provider. Clean-up logic in the autoscaler compares machines to nodes to find out + // machines at provider which could not get registered as Kubernetes nodes. With cluster-api as a + // generic out-of-tree provider for autoscaler, this field is required by autoscaler to be + // able to have a provider view of the list of machines. Another list of nodes is queried from the k8s apiserver + // and then a comparison is done to find out unregistered machines and are marked for delete. + // This field will be set by the actuators and consumed by higher level entities like autoscaler that will + // be interfacing with cluster-api as generic provider. + // +optional + ProviderID *string `json:"providerID,omitempty"` + + // version defines the desired Kubernetes version. + // This field is meant to be optionally used by bootstrap providers. + // +optional + Version *string `json:"version,omitempty"` + + // failureDomain is the failure domain the machine will be created in. + // Must match a key in the FailureDomains map stored on the cluster object. + // +optional + FailureDomain *string `json:"failureDomain,omitempty"` + + // Config for machine. contains cluster version, binary version, etc. + // + optional + Config runtime.RawExtension `json:"config,omitempty"` +} + +// KKMachineStatus defines the observed state of KKMachine. +type KKMachineStatus struct { + // Ready is true when the provider resource is ready. + // +optional + Ready bool `json:"ready,omitempty"` + + // FailureReason will be set in the event that there is a terminal problem + // +optional + FailureReason KKMachineFailedReason `json:"failureReason,omitempty"` + + // FailureMessage will be set in the event that there is a terminal problem + // +optional + FailureMessage string `json:"failureMessage,omitempty"` + + // certificatesExpiryDate is the expiry date of the machine certificates. + // This value is only set for control plane machines. + // +optional + CertificatesExpiryDate *metav1.Time `json:"certificatesExpiryDate,omitempty"` + + // Conditions defines current service state of the KKMachine. + // +optional + Conditions clusterv1beta1.Conditions `json:"conditions,omitempty"` +} + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +k8s:openapi-gen=true +// +kubebuilder:resource:scope=Namespaced,categories=cluster-api,shortName=kkm +// +kubebuilder:subresource:status +// +kubebuilder:metadata:labels="cluster.x-k8s.io/v1beta1=v1beta1" +// +kubebuilder:printcolumn:name="Cluster",type="string",JSONPath=".metadata.labels.cluster\\.x-k8s\\.io/cluster-name",description="Cluster to which this KKMachine belongs" +// +kubebuilder:printcolumn:name="ProviderID",type="string",JSONPath=".spec.providerID",description="the providerID for the machine" +// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.ready",description="Machine ready status" +// +kubebuilder:printcolumn:name="Machine",type="string",JSONPath=".metadata.ownerReferences[?(@.kind==\"Machine\")].name",description="Machine object which owns with this KKMachine" + +// KKMachine resource maps a machine instance, manage and reconcile machine status. +type KKMachine struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec KKMachineSpec `json:"spec,omitempty"` + Status KKMachineStatus `json:"status,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// KKMachineList of KKMachine +type KKMachineList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []KKMachine `json:"items"` +} + +func init() { + SchemeBuilder.Register(&KKMachine{}, &KKMachineList{}) +} diff --git a/api/capkk/infrastructure/v1beta1/kkmachinetemplate_types.go b/api/capkk/infrastructure/v1beta1/kkmachinetemplate_types.go new file mode 100644 index 000000000..723fa6fe9 --- /dev/null +++ b/api/capkk/infrastructure/v1beta1/kkmachinetemplate_types.go @@ -0,0 +1,66 @@ +/* +Copyright 2024 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +// KKMachineTemplateSpec defines the desired state of KKMachineTemplate. +type KKMachineTemplateSpec struct { + Template KKMachineTemplateResource `json:"template"` +} + +// KKMachineTemplateResource describes the data needed to create a KKMachine from a template. +type KKMachineTemplateResource struct { + // Standard object's metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + // +optional + ObjectMeta clusterv1beta1.ObjectMeta `json:"metadata,omitempty"` + + // Spec is the specification of the desired behavior of the machine. + Spec KKMachineSpec `json:"spec"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=kkmachinetemplates,scope=Namespaced,categories=cluster-api,shortName=kkmt +// +kubebuilder:storageversion +// +kubebuilder:metadata:labels="cluster.x-k8s.io/v1beta1=v1beta1" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of KKMachineTemplate" +// +k8s:defaulter-gen=true + +// KKMachineTemplate is the Schema for the kkmachinetemplates API +type KKMachineTemplate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec KKMachineTemplateSpec `json:"spec,omitempty"` +} + +// +kubebuilder:object:root=true + +// KKMachineTemplateList contains a list of KKMachineTemplate +type KKMachineTemplateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []KKMachineTemplate `json:"items"` +} + +func init() { + SchemeBuilder.Register(&KKMachineTemplate{}, &KKMachineTemplateList{}) +} diff --git a/api/capkk/infrastructure/v1beta1/register.go b/api/capkk/infrastructure/v1beta1/register.go new file mode 100644 index 000000000..701ae5c3c --- /dev/null +++ b/api/capkk/infrastructure/v1beta1/register.go @@ -0,0 +1,36 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1 contains API Schema definitions for the kubekey v1 API group +// +k8s:deepcopy-gen=package,register +// +groupName=infrastructure.cluster.x-k8s.io +package v1beta1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // SchemeGroupVersion is group version used to register these objects + SchemeGroupVersion = schema.GroupVersion{Group: "infrastructure.cluster.x-k8s.io", Version: "v1beta1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: SchemeGroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/api/capkk/infrastructure/v1beta1/zz_generated.deepcopy.go b/api/capkk/infrastructure/v1beta1/zz_generated.deepcopy.go new file mode 100644 index 000000000..7ed14e95d --- /dev/null +++ b/api/capkk/infrastructure/v1beta1/zz_generated.deepcopy.go @@ -0,0 +1,373 @@ +//go:build !ignore_autogenerated + +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1beta1 + +import ( + "k8s.io/apimachinery/pkg/runtime" + apiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InventoryHost) DeepCopyInto(out *InventoryHost) { + *out = *in + out.Connector = in.Connector + in.Vars.DeepCopyInto(&out.Vars) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InventoryHost. +func (in *InventoryHost) DeepCopy() *InventoryHost { + if in == nil { + return nil + } + out := new(InventoryHost) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InventoryHostConnector) DeepCopyInto(out *InventoryHostConnector) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InventoryHostConnector. +func (in *InventoryHostConnector) DeepCopy() *InventoryHostConnector { + if in == nil { + return nil + } + out := new(InventoryHostConnector) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KKCluster) DeepCopyInto(out *KKCluster) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKCluster. +func (in *KKCluster) DeepCopy() *KKCluster { + if in == nil { + return nil + } + out := new(KKCluster) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *KKCluster) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KKClusterList) DeepCopyInto(out *KKClusterList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]KKCluster, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKClusterList. +func (in *KKClusterList) DeepCopy() *KKClusterList { + if in == nil { + return nil + } + out := new(KKClusterList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *KKClusterList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KKClusterSpec) DeepCopyInto(out *KKClusterSpec) { + *out = *in + if in.InventoryHosts != nil { + in, out := &in.InventoryHosts, &out.InventoryHosts + *out = make([]InventoryHost, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKClusterSpec. +func (in *KKClusterSpec) DeepCopy() *KKClusterSpec { + if in == nil { + return nil + } + out := new(KKClusterSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KKClusterStatus) DeepCopyInto(out *KKClusterStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(apiv1beta1.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKClusterStatus. +func (in *KKClusterStatus) DeepCopy() *KKClusterStatus { + if in == nil { + return nil + } + out := new(KKClusterStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KKMachine) DeepCopyInto(out *KKMachine) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachine. +func (in *KKMachine) DeepCopy() *KKMachine { + if in == nil { + return nil + } + out := new(KKMachine) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *KKMachine) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KKMachineList) DeepCopyInto(out *KKMachineList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]KKMachine, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachineList. +func (in *KKMachineList) DeepCopy() *KKMachineList { + if in == nil { + return nil + } + out := new(KKMachineList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *KKMachineList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KKMachineSpec) DeepCopyInto(out *KKMachineSpec) { + *out = *in + if in.Roles != nil { + in, out := &in.Roles, &out.Roles + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ProviderID != nil { + in, out := &in.ProviderID, &out.ProviderID + *out = new(string) + **out = **in + } + if in.Version != nil { + in, out := &in.Version, &out.Version + *out = new(string) + **out = **in + } + if in.FailureDomain != nil { + in, out := &in.FailureDomain, &out.FailureDomain + *out = new(string) + **out = **in + } + in.Config.DeepCopyInto(&out.Config) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachineSpec. +func (in *KKMachineSpec) DeepCopy() *KKMachineSpec { + if in == nil { + return nil + } + out := new(KKMachineSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KKMachineStatus) DeepCopyInto(out *KKMachineStatus) { + *out = *in + if in.CertificatesExpiryDate != nil { + in, out := &in.CertificatesExpiryDate, &out.CertificatesExpiryDate + *out = (*in).DeepCopy() + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(apiv1beta1.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachineStatus. +func (in *KKMachineStatus) DeepCopy() *KKMachineStatus { + if in == nil { + return nil + } + out := new(KKMachineStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KKMachineTemplate) DeepCopyInto(out *KKMachineTemplate) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachineTemplate. +func (in *KKMachineTemplate) DeepCopy() *KKMachineTemplate { + if in == nil { + return nil + } + out := new(KKMachineTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *KKMachineTemplate) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KKMachineTemplateList) DeepCopyInto(out *KKMachineTemplateList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]KKMachineTemplate, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachineTemplateList. +func (in *KKMachineTemplateList) DeepCopy() *KKMachineTemplateList { + if in == nil { + return nil + } + out := new(KKMachineTemplateList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *KKMachineTemplateList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KKMachineTemplateResource) DeepCopyInto(out *KKMachineTemplateResource) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachineTemplateResource. +func (in *KKMachineTemplateResource) DeepCopy() *KKMachineTemplateResource { + if in == nil { + return nil + } + out := new(KKMachineTemplateResource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KKMachineTemplateSpec) DeepCopyInto(out *KKMachineTemplateSpec) { + *out = *in + in.Template.DeepCopyInto(&out.Template) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachineTemplateSpec. +func (in *KKMachineTemplateSpec) DeepCopy() *KKMachineTemplateSpec { + if in == nil { + return nil + } + out := new(KKMachineTemplateSpec) + in.DeepCopyInto(out) + return out +} diff --git a/api/core/v1/config_types.go b/api/core/v1/config_types.go new file mode 100644 index 000000000..232be669a --- /dev/null +++ b/api/core/v1/config_types.go @@ -0,0 +1,79 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "encoding/json" + + "github.com/cockroachdb/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" +) + +// Config store global vars for playbook. +type Config struct { + metav1.TypeMeta `json:",inline"` + + Spec runtime.RawExtension `json:"spec,omitempty"` +} + +// UnmarshalJSON decodes spec.Raw into spec.Object +func (c *Config) UnmarshalJSON(data []byte) error { + type Alias Config + aux := &Alias{} + if err := json.Unmarshal(data, aux); err != nil { + return errors.Wrap(err, "failed to unmarshal config") + } + *c = Config(*aux) + + // Decode spec.Raw into spec.Object if it's not already set + objMap := make(map[string]any) + if len(c.Spec.Raw) > 0 && c.Spec.Object == nil { + if err := json.Unmarshal(c.Spec.Raw, &objMap); err != nil { + return errors.Wrap(err, "failed to unmarshal spec.Raw") + } + } + c.Spec.Object = &unstructured.Unstructured{Object: objMap} + + return nil +} + +// MarshalJSON ensures spec.Object is converted back to spec.Raw +func (c *Config) MarshalJSON() ([]byte, error) { + // Ensure spec.Object is serialized into spec.Raw + if c.Spec.Object != nil { + raw, err := json.Marshal(c.Spec.Object) + if err != nil { + return nil, errors.Wrap(err, "failed to marshal spec.Object") + } + c.Spec.Raw = raw + } + + type Alias Config + return json.Marshal((*Alias)(c)) +} + +// Value returns the underlying map[string]any from the Config's unstructured Object. +// This provides direct access to the config values stored in Spec.Object. +func (c *Config) Value() map[string]any { + if c.Spec.Object == nil { + c.Spec.Object = &unstructured.Unstructured{Object: make(map[string]any)} + } + + return c.Spec.Object.(*unstructured.Unstructured).Object +} diff --git a/api/core/v1/inventory_types.go b/api/core/v1/inventory_types.go new file mode 100644 index 000000000..05707f56f --- /dev/null +++ b/api/core/v1/inventory_types.go @@ -0,0 +1,131 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +const ( + // InventoryCAPKKFinalizer is used to waiting ref playbook compelete when inventory is deleted. + InventoryCAPKKFinalizer = "inventory.kubekey.kubesphere.io/capkk" + + // HostCheckPlaybookAnnotation store which playbook is used to check hosts. + HostCheckPlaybookAnnotation = "playbook.kubekey.kubesphere.io/host-check" +) + +// InventoryPhase of inventory. it's always use in capkk to judge if host has checked. +type InventoryPhase string + +const ( + // InventoryPhasePending inventory has created but has never been checked once + InventoryPhasePending InventoryPhase = "Pending" + // InventoryPhaseRunning inventory host_check playbook is running. + InventoryPhaseRunning InventoryPhase = "Running" + // InventoryPhaseReady inventory host_check playbook run successfully. + InventoryPhaseSucceeded InventoryPhase = "Succeeded" + // InventoryPhaseReady inventory host_check playbook run check failed. + InventoryPhaseFailed InventoryPhase = "Failed" +) + +// InventoryHost of Inventory +type InventoryHost map[string]runtime.RawExtension + +// InventoryGroup of Inventory +type InventoryGroup struct { + Groups []string `json:"groups,omitempty"` + Hosts []string `json:"hosts"` + Vars runtime.RawExtension `json:"vars,omitempty"` +} + +// InventorySpec of Inventory +type InventorySpec struct { + // Hosts is all nodes + Hosts InventoryHost `json:"hosts"` + // Vars for all host. the priority for vars is: host vars > group vars > inventory vars + // +optional + // +kubebuilder:pruning:PreserveUnknownFields + Vars runtime.RawExtension `json:"vars,omitempty"` + // Groups nodes. a group contains repeated nodes + // +optional + Groups map[string]InventoryGroup `json:"groups,omitempty"` +} + +// InventoryStatus of Inventory +type InventoryStatus struct { + // Ready is the inventory ready to be used. + Ready bool `json:"ready,omitempty"` + // Phase is the inventory phase. + Phase InventoryPhase `json:"phase,omitempty"` +} + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +k8s:openapi-gen=true +// +kubebuilder:resource:scope=Namespaced +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase",description="Status of inventory" + +// Inventory store hosts vars for playbook. +type Inventory struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec InventorySpec `json:"spec,omitempty"` + Status InventoryStatus `json:"status,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// InventoryList of Inventory +type InventoryList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Inventory `json:"items"` +} + +// GetHostsFromGroup flatten a specific `Inventory` group with de-duplication. +func GetHostsFromGroup(inv *Inventory, groupName string, unavailableHosts, unavailableGroups map[string]struct{}) []string { + var hosts = make([]string, 0) + if v, ok := inv.Spec.Groups[groupName]; ok { + unavailableGroups[groupName] = struct{}{} + for _, cg := range v.Groups { + if _, exist := unavailableGroups[cg]; !exist { + unavailableGroups[cg] = struct{}{} + hosts = append(hosts, GetHostsFromGroup(inv, cg, unavailableHosts, unavailableGroups)...) + } + } + + validHosts := make([]string, 0) + for _, hostname := range v.Hosts { + if _, ok := inv.Spec.Hosts[hostname]; ok { + if _, exist := unavailableHosts[hostname]; !exist { + unavailableHosts[hostname] = struct{}{} + validHosts = append(validHosts, hostname) + } + } + } + hosts = append(hosts, validHosts...) + } + + return hosts +} + +func init() { + SchemeBuilder.Register(&Inventory{}, &InventoryList{}) +} diff --git a/api/core/v1/playbook_types.go b/api/core/v1/playbook_types.go new file mode 100644 index 000000000..d3ba2f89e --- /dev/null +++ b/api/core/v1/playbook_types.go @@ -0,0 +1,174 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +const ( + // BuiltinsProjectAnnotation use builtins project of KubeKey + BuiltinsProjectAnnotation = "kubekey.kubesphere.io/builtins-project" + + // PlaybookCompletedFinalizer will be removed after the Playbook is completed. + PlaybookCompletedFinalizer = "kubekey.kubesphere.io/playbook-completed" +) + +// PlaybookPhase of Playbook +type PlaybookPhase string + +const ( + // PlaybookPhasePending of Playbook. Playbook has created but not deal + PlaybookPhasePending PlaybookPhase = "Pending" + // PlaybookPhaseRunning of Playbook. deal Playbook. + PlaybookPhaseRunning PlaybookPhase = "Running" + // PlaybookPhaseFailed of Playbook. once Task run failed. + PlaybookPhaseFailed PlaybookPhase = "Failed" + // PlaybookPhaseSucceed of Playbook. all Tasks run success. + PlaybookPhaseSucceeded PlaybookPhase = "Succeeded" +) + +// PlaybookFailedReason is the reason why a Playbook failed. +type PlaybookFailedReason string + +const ( + // PlaybookFailedReasonUnknown is the default failed reason. + PlaybookFailedReasonUnknown PlaybookFailedReason = "unknown" + // PlaybookFailedReasonPodFailed pod exec failed. + PlaybookFailedReasonPodFailed PlaybookFailedReason = "pod executor failed" + // PlaybookFailedReasonTaskFailed task exec failed. + PlaybookFailedReasonTaskFailed PlaybookFailedReason = "task executor failed" +) + +// PlaybookSpec defines the desired state of Playbook. +type PlaybookSpec struct { + // Project is storage for executable packages + // +optional + Project PlaybookProject `json:"project,omitempty"` + // Playbook which to execute. + Playbook string `json:"playbook"` + // InventoryRef is the node configuration for playbook + // +optional + InventoryRef *corev1.ObjectReference `json:"inventoryRef,omitempty"` + // Config is the global variable configuration for playbook + // +optional + Config Config `json:"config,omitempty"` + // Tags is the tags of playbook which to execute + // +optional + Tags []string `json:"tags,omitempty"` + // SkipTags is the tags of playbook which skip execute + // +optional + SkipTags []string `json:"skipTags,omitempty"` + // Volumes in job pod. + // +optional + Volumes []corev1.Volume `json:"workVolume,omitempty"` + // VolumeMounts in job pod. + // +optional + VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"` + // ServiceAccountName is the name of the ServiceAccount to use to run this pod. + // More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + // +optional + ServiceAccountName string `json:"serviceAccountName,omitempty"` +} + +// PlaybookProject respect which playbook store. +type PlaybookProject struct { + // Addr is the storage for executable packages (in Ansible file format). + // When starting with http or https, it will be obtained from a Git repository. + // When starting with file path, it will be obtained from the local path. + // +optional + Addr string `json:"addr,omitempty"` + // Name is the project name base project + // +optional + Name string `json:"name,omitempty"` + // Branch is the git branch of the git Addr. + // +optional + Branch string `json:"branch,omitempty"` + // Tag is the git branch of the git Addr. + // +optional + Tag string `json:"tag,omitempty"` + // InsecureSkipTLS skip tls or not when git addr is https. + // +optional + InsecureSkipTLS bool `json:"insecureSkipTLS,omitempty"` + // Token of Authorization for http request + // +optional + Token string `json:"token,omitempty"` +} + +// PlaybookStatus defines the observed state of Playbook. +type PlaybookStatus struct { + // Statistics statistics of task counts + Statistics PlaybookStatistics `json:"statistics,omitempty"` + // Result will record the results detail. + Result runtime.RawExtension `json:"result,omitempty"` + // Phase of playbook. + Phase PlaybookPhase `json:"phase,omitempty"` + // FailureReason will be set in the event that there is a terminal problem + // +optional + FailureReason PlaybookFailedReason `json:"failureReason,omitempty"` + // FailureMessage will be set in the event that there is a terminal problem + // +optional + FailureMessage string `json:"failureMessage,omitempty"` +} + +// PlaybookStatistics contains statistics of task counts. +type PlaybookStatistics struct { + // Total number of tasks + Total int `json:"total,omitempty"` + // Number of successful tasks + Success int `json:"success,omitempty"` + // Number of failed tasks + Failed int `json:"failed,omitempty"` + // Number of ignored tasks + Ignored int `json:"ignored,omitempty"` +} + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +k8s:openapi-gen=true +// +kubebuilder:resource:scope=Namespaced +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Playbook",type="string",JSONPath=".spec.playbook" +// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase" +// +kubebuilder:printcolumn:name="Total",type="integer",JSONPath=".status.taskResult.total" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +// Playbook is the Schema for the playbooks API. +// Playbook resource executor a playbook. +type Playbook struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec PlaybookSpec `json:"spec,omitempty"` + Status PlaybookStatus `json:"status,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// PlaybookList contains a list of Playbook. +type PlaybookList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Playbook `json:"items"` +} + +// Register Playbook and PlaybookList types with the scheme. +func init() { + SchemeBuilder.Register(&Playbook{}, &PlaybookList{}) +} diff --git a/api/core/v1/register.go b/api/core/v1/register.go new file mode 100644 index 000000000..7eb22a496 --- /dev/null +++ b/api/core/v1/register.go @@ -0,0 +1,36 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1 contains API Schema definitions for the kubekey v1 API group +// +k8s:deepcopy-gen=package,register +// +groupName=kubekey.kubesphere.io +package v1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // SchemeGroupVersion is group version used to register these objects + SchemeGroupVersion = schema.GroupVersion{Group: "kubekey.kubesphere.io", Version: "v1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: SchemeGroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/api/core/v1/zz_generated.deepcopy.go b/api/core/v1/zz_generated.deepcopy.go new file mode 100644 index 000000000..9820019ff --- /dev/null +++ b/api/core/v1/zz_generated.deepcopy.go @@ -0,0 +1,346 @@ +//go:build !ignore_autogenerated + +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1 + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Config) DeepCopyInto(out *Config) { + *out = *in + out.TypeMeta = in.TypeMeta + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Config. +func (in *Config) DeepCopy() *Config { + if in == nil { + return nil + } + out := new(Config) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Inventory) DeepCopyInto(out *Inventory) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Inventory. +func (in *Inventory) DeepCopy() *Inventory { + if in == nil { + return nil + } + out := new(Inventory) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Inventory) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InventoryGroup) DeepCopyInto(out *InventoryGroup) { + *out = *in + if in.Groups != nil { + in, out := &in.Groups, &out.Groups + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Hosts != nil { + in, out := &in.Hosts, &out.Hosts + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.Vars.DeepCopyInto(&out.Vars) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InventoryGroup. +func (in *InventoryGroup) DeepCopy() *InventoryGroup { + if in == nil { + return nil + } + out := new(InventoryGroup) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in InventoryHost) DeepCopyInto(out *InventoryHost) { + { + in := &in + *out = make(InventoryHost, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InventoryHost. +func (in InventoryHost) DeepCopy() InventoryHost { + if in == nil { + return nil + } + out := new(InventoryHost) + in.DeepCopyInto(out) + return *out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InventoryList) DeepCopyInto(out *InventoryList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Inventory, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InventoryList. +func (in *InventoryList) DeepCopy() *InventoryList { + if in == nil { + return nil + } + out := new(InventoryList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *InventoryList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InventorySpec) DeepCopyInto(out *InventorySpec) { + *out = *in + if in.Hosts != nil { + in, out := &in.Hosts, &out.Hosts + *out = make(InventoryHost, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } + in.Vars.DeepCopyInto(&out.Vars) + if in.Groups != nil { + in, out := &in.Groups, &out.Groups + *out = make(map[string]InventoryGroup, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InventorySpec. +func (in *InventorySpec) DeepCopy() *InventorySpec { + if in == nil { + return nil + } + out := new(InventorySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InventoryStatus) DeepCopyInto(out *InventoryStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InventoryStatus. +func (in *InventoryStatus) DeepCopy() *InventoryStatus { + if in == nil { + return nil + } + out := new(InventoryStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Playbook) DeepCopyInto(out *Playbook) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Playbook. +func (in *Playbook) DeepCopy() *Playbook { + if in == nil { + return nil + } + out := new(Playbook) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Playbook) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PlaybookList) DeepCopyInto(out *PlaybookList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Playbook, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlaybookList. +func (in *PlaybookList) DeepCopy() *PlaybookList { + if in == nil { + return nil + } + out := new(PlaybookList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PlaybookList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PlaybookProject) DeepCopyInto(out *PlaybookProject) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlaybookProject. +func (in *PlaybookProject) DeepCopy() *PlaybookProject { + if in == nil { + return nil + } + out := new(PlaybookProject) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PlaybookSpec) DeepCopyInto(out *PlaybookSpec) { + *out = *in + out.Project = in.Project + if in.InventoryRef != nil { + in, out := &in.InventoryRef, &out.InventoryRef + *out = new(corev1.ObjectReference) + **out = **in + } + in.Config.DeepCopyInto(&out.Config) + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.SkipTags != nil { + in, out := &in.SkipTags, &out.SkipTags + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Volumes != nil { + in, out := &in.Volumes, &out.Volumes + *out = make([]corev1.Volume, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.VolumeMounts != nil { + in, out := &in.VolumeMounts, &out.VolumeMounts + *out = make([]corev1.VolumeMount, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlaybookSpec. +func (in *PlaybookSpec) DeepCopy() *PlaybookSpec { + if in == nil { + return nil + } + out := new(PlaybookSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PlaybookStatistics) DeepCopyInto(out *PlaybookStatistics) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlaybookStatistics. +func (in *PlaybookStatistics) DeepCopy() *PlaybookStatistics { + if in == nil { + return nil + } + out := new(PlaybookStatistics) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PlaybookStatus) DeepCopyInto(out *PlaybookStatus) { + *out = *in + out.Statistics = in.Statistics + in.Result.DeepCopyInto(&out.Result) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlaybookStatus. +func (in *PlaybookStatus) DeepCopy() *PlaybookStatus { + if in == nil { + return nil + } + out := new(PlaybookStatus) + in.DeepCopyInto(out) + return out +} diff --git a/api/core/v1alpha1/conversion.go b/api/core/v1alpha1/conversion.go new file mode 100644 index 000000000..31c9861ef --- /dev/null +++ b/api/core/v1alpha1/conversion.go @@ -0,0 +1,32 @@ +package v1alpha1 + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" +) + +// supportedFields defines the set of field labels that are allowed for field selector queries +// on the Task resource. Only these fields can be used in field selectors for filtering. +var supportedFields = sets.NewString( + "metadata.name", + "metadata.namespace", + "playbook.name", + "playbook.uid", +) + +// RegisterFieldLabelConversion registers a field label conversion function for the Task resource. +// This function ensures that only supported field labels can be used in field selectors. +// If an unsupported field label is used, an error is returned. +func RegisterFieldLabelConversion(scheme *runtime.Scheme) error { + return scheme.AddFieldLabelConversionFunc( + SchemeGroupVersion.WithKind("Task"), + func(label, value string) (string, string, error) { + if !supportedFields.Has(label) { + return "", "", fmt.Errorf("field label %q is not supported", label) + } + return label, value, nil + }, + ) +} diff --git a/api/core/v1alpha1/register.go b/api/core/v1alpha1/register.go new file mode 100644 index 000000000..986c483a3 --- /dev/null +++ b/api/core/v1alpha1/register.go @@ -0,0 +1,37 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha1 is the internal version, should not register in kubernetes +// +k8s:deepcopy-gen=package,register +// +groupName=kubekey.kubesphere.io +// +kubebuilder:skip +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // SchemeGroupVersion is group version used to register these objects + SchemeGroupVersion = schema.GroupVersion{Group: "kubekey.kubesphere.io", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: SchemeGroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/api/core/v1alpha1/task_types.go b/api/core/v1alpha1/task_types.go new file mode 100644 index 000000000..2896699a3 --- /dev/null +++ b/api/core/v1alpha1/task_types.go @@ -0,0 +1,122 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// TaskPhase of Task +type TaskPhase string + +const ( + // TaskPhasePending of Task. Task has created but not deal + TaskPhasePending TaskPhase = "Pending" + // TaskPhaseRunning of Task. deal Task + TaskPhaseRunning TaskPhase = "Running" + // TaskPhaseSuccess of Task. Module of Task run success in each hosts. + TaskPhaseSuccess TaskPhase = "Success" + // TaskPhaseFailed of Task. once host run failed. + TaskPhaseFailed TaskPhase = "Failed" + // TaskPhaseIgnored of Task. once host run failed and set ignore_errors. + TaskPhaseIgnored TaskPhase = "Ignored" +) + +const ( + // TaskAnnotationRelativePath is the relative dir of task in project. + TaskAnnotationRelativePath = "kubesphere.io/rel-path" +) + +// TaskSpec of Task +type TaskSpec struct { + Name string `json:"name,omitempty"` + Hosts []string `json:"hosts,omitempty"` + DelegateTo string `yaml:"delegate_to,omitempty"` + IgnoreError *bool `json:"ignoreError,omitempty"` + Retries int `json:"retries,omitempty"` + + When []string `json:"when,omitempty"` + FailedWhen []string `json:"failedWhen,omitempty"` + Loop runtime.RawExtension `json:"loop,omitempty"` + + Module Module `json:"module,omitempty"` + Register string `json:"register,omitempty"` + RegisterType string `json:"register_type,omitempty"` +} + +// Module of Task +type Module struct { + Name string `json:"name,omitempty"` + Args runtime.RawExtension `json:"args,omitempty"` +} + +// TaskStatus of Task +type TaskStatus struct { + RestartCount int `json:"restartCount,omitempty"` + Phase TaskPhase `json:"phase,omitempty"` + HostResults []TaskHostResult `json:"hostResults,omitempty"` +} + +// TaskHostResult each host result for task +type TaskHostResult struct { + Host string `json:"host,omitempty"` + Stdout string `json:"stdout,omitempty"` + StdErr string `json:"stdErr,omitempty"` + Error string `json:"error,omitempty"` +} + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:resource:scope=Namespaced + +// Task of playbook +type Task struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec TaskSpec `json:"spec,omitempty"` + Status TaskStatus `json:"status,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// TaskList for Task +type TaskList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Task `json:"items"` +} + +// IsComplete if Task IsSucceed or IsFailed +func (t Task) IsComplete() bool { + return t.IsSucceed() || t.IsFailed() +} + +// IsSucceed if Task.Status.Phase TaskPhaseSuccess or TaskPhaseIgnored +func (t Task) IsSucceed() bool { + return t.Status.Phase == TaskPhaseSuccess || t.Status.Phase == TaskPhaseIgnored +} + +// IsFailed Task.Status.Phase is failed when reach the retries +func (t Task) IsFailed() bool { + return t.Status.Phase == TaskPhaseFailed && t.Spec.Retries <= t.Status.RestartCount +} + +func init() { + SchemeBuilder.Register(&Task{}, &TaskList{}) +} diff --git a/api/core/v1alpha1/zz_generated.deepcopy.go b/api/core/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 000000000..b00b302a5 --- /dev/null +++ b/api/core/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,172 @@ +//go:build !ignore_autogenerated + +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Module) DeepCopyInto(out *Module) { + *out = *in + in.Args.DeepCopyInto(&out.Args) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Module. +func (in *Module) DeepCopy() *Module { + if in == nil { + return nil + } + out := new(Module) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Task) DeepCopyInto(out *Task) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Task. +func (in *Task) DeepCopy() *Task { + if in == nil { + return nil + } + out := new(Task) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Task) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TaskHostResult) DeepCopyInto(out *TaskHostResult) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TaskHostResult. +func (in *TaskHostResult) DeepCopy() *TaskHostResult { + if in == nil { + return nil + } + out := new(TaskHostResult) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TaskList) DeepCopyInto(out *TaskList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Task, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TaskList. +func (in *TaskList) DeepCopy() *TaskList { + if in == nil { + return nil + } + out := new(TaskList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TaskList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TaskSpec) DeepCopyInto(out *TaskSpec) { + *out = *in + if in.Hosts != nil { + in, out := &in.Hosts, &out.Hosts + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.IgnoreError != nil { + in, out := &in.IgnoreError, &out.IgnoreError + *out = new(bool) + **out = **in + } + if in.When != nil { + in, out := &in.When, &out.When + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.FailedWhen != nil { + in, out := &in.FailedWhen, &out.FailedWhen + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.Loop.DeepCopyInto(&out.Loop) + in.Module.DeepCopyInto(&out.Module) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TaskSpec. +func (in *TaskSpec) DeepCopy() *TaskSpec { + if in == nil { + return nil + } + out := new(TaskSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TaskStatus) DeepCopyInto(out *TaskStatus) { + *out = *in + if in.HostResults != nil { + in, out := &in.HostResults, &out.HostResults + *out = make([]TaskHostResult, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TaskStatus. +func (in *TaskStatus) DeepCopy() *TaskStatus { + if in == nil { + return nil + } + out := new(TaskStatus) + in.DeepCopyInto(out) + return out +} diff --git a/api/go.mod b/api/go.mod new file mode 100644 index 000000000..1b3940a43 --- /dev/null +++ b/api/go.mod @@ -0,0 +1,53 @@ +module github.com/kubesphere/kubekey/api + +go 1.23.3 + +require ( + github.com/cockroachdb/errors v1.11.3 + github.com/stretchr/testify v1.9.0 + gopkg.in/yaml.v3 v3.0.1 + k8s.io/api v0.31.3 + k8s.io/apimachinery v0.31.3 + sigs.k8s.io/cluster-api v1.9.2 + sigs.k8s.io/controller-runtime v0.19.3 +) + +require ( + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/redact v1.1.5 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.12.1 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/getsentry/sentry-go v0.27.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.4 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + golang.org/x/net v0.32.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect + google.golang.org/protobuf v1.35.1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + k8s.io/apiextensions-apiserver v0.31.3 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect +) diff --git a/api/go.sum b/api/go.sum new file mode 100644 index 000000000..46ac8c894 --- /dev/null +++ b/api/go.sum @@ -0,0 +1,162 @@ +github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= +github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= +github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= +github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= +github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.36.0 h1:Pb12RlruUtj4XUuPUqeEWc6j5DkVVVA49Uf6YLfC95Y= +github.com/onsi/gomega v1.36.0/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= +golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.31.3 h1:umzm5o8lFbdN/hIXbrK9oRpOproJO62CV1zqxXrLgk8= +k8s.io/api v0.31.3/go.mod h1:UJrkIp9pnMOI9K2nlL6vwpxRzzEX5sWgn8kGQe92kCE= +k8s.io/apiextensions-apiserver v0.31.3 h1:+GFGj2qFiU7rGCsA5o+p/rul1OQIq6oYpQw4+u+nciE= +k8s.io/apiextensions-apiserver v0.31.3/go.mod h1:2DSpFhUZZJmn/cr/RweH1cEVVbzFw9YBu4T+U3mf1e4= +k8s.io/apimachinery v0.31.3 h1:6l0WhcYgasZ/wk9ktLq5vLaoXJJr5ts6lkaQzgeYPq4= +k8s.io/apimachinery v0.31.3/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/cluster-api v1.9.2 h1:4nUcIg/nOccn7/O1FF1IJaxQqjOxl+gH4ejQ9D/P+l8= +sigs.k8s.io/cluster-api v1.9.2/go.mod h1:pkFqVPq0ELlJgyDjgqpb4MU1XnWEi98B2q3DbEjC4ww= +sigs.k8s.io/controller-runtime v0.19.3 h1:XO2GvC9OPftRst6xWCpTgBZO04S2cbp0Qqkj8bX1sPw= +sigs.k8s.io/controller-runtime v0.19.3/go.mod h1:j4j87DqtsThvwTv5/Tc5NFRyyF/RF0ip4+62tbTSIUM= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/api/project/v1/base.go b/api/project/v1/base.go new file mode 100644 index 000000000..6f4a6001c --- /dev/null +++ b/api/project/v1/base.go @@ -0,0 +1,69 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import "gopkg.in/yaml.v3" + +// Base defined in project. +type Base struct { + Name string `yaml:"name,omitempty"` + + // connection/transport + Connection string `yaml:"connection,omitempty"` + Port int `yaml:"port,omitempty"` + RemoteUser string `yaml:"remote_user,omitempty"` + + // variables + Vars Vars `yaml:"vars,omitempty"` + + // module default params + //ModuleDefaults []map[string]map[string]any `yaml:"module_defaults,omitempty"` + + // flags and misc. settings + Environment []map[string]string `yaml:"environment,omitempty"` + NoLog bool `yaml:"no_log,omitempty"` + RunOnce bool `yaml:"run_once,omitempty"` + IgnoreErrors *bool `yaml:"ignore_errors,omitempty"` + CheckMode bool `yaml:"check_mode,omitempty"` + Diff bool `yaml:"diff,omitempty"` + AnyErrorsFatal bool `yaml:"any_errors_fatal,omitempty"` + Throttle int `yaml:"throttle,omitempty"` + Timeout int `yaml:"timeout,omitempty"` + + // Debugger invoke a debugger on tasks + Debugger string `yaml:"debugger,omitempty"` + + // privilege escalation + Become bool `yaml:"become,omitempty"` + BecomeMethod string `yaml:"become_method,omitempty"` + BecomeUser string `yaml:"become_user,omitempty"` + BecomeFlags string `yaml:"become_flags,omitempty"` + BecomeExe string `yaml:"become_exe,omitempty"` +} + +// Vars is a custom type to hold a list of YAML nodes representing variables. +// This allows for flexible unmarshalling of various YAML structures into Vars. +type Vars struct { + Nodes []yaml.Node +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface for Vars. +// It appends the unmarshalled YAML node to the Vars.Nodes slice. +func (v *Vars) UnmarshalYAML(node *yaml.Node) error { + v.Nodes = append(v.Nodes, *node) + return nil +} diff --git a/api/project/v1/block.go b/api/project/v1/block.go new file mode 100644 index 000000000..41bd847cb --- /dev/null +++ b/api/project/v1/block.go @@ -0,0 +1,156 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "fmt" + "reflect" + "strings" + + "github.com/cockroachdb/errors" + "gopkg.in/yaml.v3" +) + +// Block defined in project. +type Block struct { + BlockBase + // If it has Block, Task should be empty + Task + IncludeTasks string `yaml:"include_tasks,omitempty"` + + BlockInfo +} + +// BlockBase defined in project. +type BlockBase struct { + Base `yaml:",inline"` + Conditional `yaml:",inline"` + CollectionSearch `yaml:",inline"` + Taggable `yaml:",inline"` + Notifiable `yaml:",inline"` + Delegable `yaml:",inline"` +} + +// BlockInfo defined in project. +type BlockInfo struct { + Block []Block `yaml:"block,omitempty"` + Rescue []Block `yaml:"rescue,omitempty"` + Always []Block `yaml:"always,omitempty"` +} + +// Task defined in project. +type Task struct { + AsyncVal int `yaml:"async,omitempty"` + ChangedWhen When `yaml:"changed_when,omitempty"` + Delay int `yaml:"delay,omitempty"` + FailedWhen When `yaml:"failed_when,omitempty"` + Loop any `yaml:"loop,omitempty"` + LoopControl LoopControl `yaml:"loop_control,omitempty"` + Poll int `yaml:"poll,omitempty"` + Register string `yaml:"register,omitempty"` + // RegisterType how to register value to variable. support: string(default), json, yaml. + RegisterType string `yaml:"register_type,omitempty"` + Retries int `yaml:"retries,omitempty"` + Until When `yaml:"until,omitempty"` + + // deprecated, used to be loop and loop_args but loop has been repurposed + //LoopWith string `yaml:"loop_with"` + + // UnknownField store undefined field + UnknownField map[string]any `yaml:"-"` +} + +// UnmarshalYAML yaml to block. +func (b *Block) UnmarshalYAML(node *yaml.Node) error { + // fill baseInfo + if err := node.Decode(&b.BlockBase); err != nil { + return errors.Wrap(err, "failed to decode block") + } + + for i := 0; i < len(node.Content); i += 2 { + keyNode := node.Content[i] + valueNode := node.Content[i+1] + + switch keyNode.Value { + case "include_tasks": + b.IncludeTasks = valueNode.Value + return nil + + case "block": + return node.Decode(&b.BlockInfo) + } + } + + if err := node.Decode(&b.Task); err != nil { + return errors.Wrap(err, "failed to decode task") + } + b.UnknownField = collectUnknownFields(node, append(getFieldNames(reflect.TypeOf(BlockBase{})), getFieldNames(reflect.TypeOf(Task{}))...)) + + return nil +} + +// collectUnknownFields traverses a YAML node and collects fields that are not in the excludeFields list. +// It returns a map where the keys are the names of the unknown fields and the values are their corresponding values. +func collectUnknownFields(node *yaml.Node, excludeFields []string) map[string]any { + unknown := make(map[string]any) + excludeSet := make(map[string]struct{}, len(excludeFields)) + for _, field := range excludeFields { + excludeSet[field] = struct{}{} + } + + for i := 0; i < len(node.Content); i += 2 { + keyNode := node.Content[i] + valueNode := node.Content[i+1] + + if _, excluded := excludeSet[keyNode.Value]; excluded { + continue + } + + var value any + if err := valueNode.Decode(&value); err == nil { + unknown[keyNode.Value] = value + } else { + unknown[keyNode.Value] = fmt.Sprintf("failed to decode: %v", err) + } + } + + return unknown +} + +// getFieldNames returns a slice of field names for a given struct type. +// It inspects the struct fields and extracts the names from the "yaml" tags. +// If a field has an "inline" tag, it recursively processes the fields of the embedded struct. +func getFieldNames(t reflect.Type) []string { + var fields []string + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + yamlTag := field.Tag.Get("yaml") + if yamlTag != "" { + if strings.Contains(yamlTag, "inline") { + inlineFields := getFieldNames(field.Type) + fields = append(fields, inlineFields...) + continue + } + tagName := strings.Split(yamlTag, ",")[0] + if tagName != "" && tagName != "-" { + fields = append(fields, tagName) + } + } + } + + return fields +} diff --git a/api/project/v1/collectionsearch.go b/api/project/v1/collectionsearch.go new file mode 100644 index 000000000..3dd2ce621 --- /dev/null +++ b/api/project/v1/collectionsearch.go @@ -0,0 +1,22 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +// CollectionSearch defined in project. +type CollectionSearch struct { + Collections []string `yaml:"collections,omitempty"` +} diff --git a/api/project/v1/conditional.go b/api/project/v1/conditional.go new file mode 100644 index 000000000..04258ef2d --- /dev/null +++ b/api/project/v1/conditional.go @@ -0,0 +1,57 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "github.com/cockroachdb/errors" + "gopkg.in/yaml.v3" +) + +// Conditional defined in project. +type Conditional struct { + When When `yaml:"when,omitempty"` +} + +// When defined in project. +type When struct { + Data []string +} + +// UnmarshalYAML yaml string to when +func (w *When) UnmarshalYAML(node *yaml.Node) error { + switch node.Kind { + case yaml.ScalarNode: + if IsTmplSyntax(node.Value) { + w.Data = []string{node.Value} + } else { + w.Data = []string{"{{ " + node.Value + " }}"} + } + case yaml.SequenceNode: + if err := node.Decode(&w.Data); err != nil { + return errors.WithStack(err) + } + for i, v := range w.Data { + if !IsTmplSyntax(v) { + w.Data[i] = ParseTmplSyntax(v) + } + } + default: + return errors.New("unsupported type, excepted string or array of strings") + } + + return nil +} diff --git a/api/project/v1/conditional_test.go b/api/project/v1/conditional_test.go new file mode 100644 index 000000000..70714fb36 --- /dev/null +++ b/api/project/v1/conditional_test.go @@ -0,0 +1,45 @@ +package v1 + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" +) + +func TestUnmarshalWhen(t *testing.T) { + testcases := []struct { + name string + content string + except []string + }{ + { + name: "test single string", + content: ` +when: .a | eq "b"`, + except: []string{ + ".a | eq \"b\"", + }, + }, + { + name: "test multi string", + content: ` +when: + - .a | eq "b" + - .b | ne "c"`, + except: []string{ + ".a | eq \"b\"", + ".b | ne \"c\"", + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + var when Conditional + err := yaml.Unmarshal([]byte(tc.content), &when) + assert.NoError(t, err) + assert.Equal(t, tc.except, when.When.Data) + }) + } +} diff --git a/api/project/v1/delegatable.go b/api/project/v1/delegatable.go new file mode 100644 index 000000000..2516a5e16 --- /dev/null +++ b/api/project/v1/delegatable.go @@ -0,0 +1,23 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +// Delegable defined in project. +type Delegable struct { + DelegateTo string `yaml:"delegate_to,omitempty"` + DelegateFacts bool `yaml:"delegate_facts,omitempty"` +} diff --git a/api/project/v1/docs.go b/api/project/v1/docs.go new file mode 100644 index 000000000..72082c649 --- /dev/null +++ b/api/project/v1/docs.go @@ -0,0 +1,188 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +// Playbook keyword in ansible: https://docs.ansible.com/ansible/latest/reference_appendices/playbooks_keywords.html#playbook-keywords +// support list (base on ansible 2.15.5) + +/** +Play ++------+------------------------+------------+ +| Row | Keyword | Support | ++------+------------------------+------------+ +| 1 | any_errors_fatal | ✘ | +| 2 | become | ✘ | +| 3 | become_exe | ✘ | +| 4 | become_flags | ✘ | +| 5 | become_method | ✘ | +| 6 | become_user | ✘ | +| 7 | check_mode | ✘ | +| 8 | collections | ✘ | +| 9 | connection | ✔︎ | +| 10 | debugger | ✘ | +| 11 | diff | ✘ | +| 12 | environment | ✘ | +| 13 | fact_path | ✘ | +| 14 | force_handlers | ✘ | +| 15 | gather_facts | ✔︎ | +| 16 | gather_subset | ✘ | +| 17 | gather_timeout | ✘ | +| 18 | handlers | ✘ | +| 19 | hosts | ✔︎ | +| 20 | ignore_errors | ✔︎ | +| 21 | ignore_unreachable | ✘ | +| 22 | max_fail_percentage | ✘ | +| 23 | module_defaults | ✘ | +| 24 | name | ✔︎ | +| 25 | no_log | ✘ | +| 26 | order | ✘ | +| 27 | port | ✘ | +| 28 | post_task | ✔︎ | +| 29 | pre_tasks | ✔︎ | +| 30 | remote_user | ✘ | +| 31 | roles | ✔︎ | +| 32 | run_once | ✔︎ | +| 33 | serial | ✔︎ | +| 34 | strategy | ✘ | +| 35 | tags | ✔︎ | +| 36 | tasks | ✔︎ | +| 37 | throttle | ✘ | +| 38 | timeout | ✘ | +| 39 | vars | ✔︎ | +| 40 | vars_files | ✘ | +| 41 | vars_prompt | ✘ | ++------+------------------------+------------+ + +Role ++------+------------------------+------------+ +| Row | Keyword | Support | ++------+------------------------+------------+ +| 1 | any_errors_fatal | ✘ | +| 2 | become | ✘ | +| 3 | become_exe | ✘ | +| 4 | become_flags | ✘ | +| 5 | become_method | ✘ | +| 6 | become_user | ✘ | +| 7 | check_mode | ✘ | +| 8 | collections | ✘ | +| 9 | connection | ✘ | +| 10 | debugger | ✘ | +| 11 | delegate_facts | ✘ | +| 12 | delegate_to | ✘ | +| 13 | diff | ✘ | +| 14 | environment | ✘ | +| 15 | ignore_errors | ✔︎ | +| 16 | ignore_unreachable | ✘ | +| 17 | max_fail_percentage | ✘ | +| 18 | module_defaults | ✘ | +| 19 | name | ✔︎ | +| 20 | no_log | ✘ | +| 21 | port | ✘ | +| 22 | remote_user | ✘ | +| 23 | run_once | ✔︎ | +| 24 | tags | ✔︎ | +| 25 | throttle | ✘ | +| 26 | timeout | ✘ | +| 27 | vars | ✔︎ | +| 28 | when | ✔︎ | ++------+------------------------+------------+ + +Block ++------+------------------------+------------+ +| Row | Keyword | Support | ++------+------------------------+------------+ +| 1 | always | ✔︎ | +| 2 | any_errors_fatal | ✘ | +| 3 | become | ✘ | +| 4 | become_exe | ✘ | +| 5 | become_flags | ✘ | +| 6 | become_method | ✘ | +| 7 | become_user | ✘ | +| 8 | block | ✔︎ | +| 9 | check_mode | ✘ | +| 10 | collections | ✘ | +| 11 | debugger | ✘ | +| 12 | delegate_facts | ✘ | +| 13 | delegate_to | ✘ | +| 14 | diff | ✘ | +| 15 | environment | ✘ | +| 16 | ignore_errors | ✔︎ | +| 17 | ignore_unreachable | ✘ | +| 18 | max_fail_percentage | ✘ | +| 19 | module_defaults | ✘ | +| 20 | name | ✔︎ | +| 21 | no_log | ✘ | +| 22 | notify | ✘ | +| 23 | port | ✘ | +| 24 | remote_user | ✘ | +| 25 | rescue | ✔︎ | +| 26 | run_once | ✘ | +| 27 | tags | ✔︎ | +| 28 | throttle | ✘ | +| 29 | timeout | ✘ | +| 30 | vars | ✔︎ | +| 31 | when | ✔︎ | ++------+------------------------+------------+ + + +Task ++------+------------------------+------------+ +| Row | Keyword | Support | ++------+------------------------+------------+ +| 1 | action | ✔︎ | +| 2 | any_errors_fatal | ✘ | +| 3 | args | ✔︎ | +| 4 | async | ✘ | +| 5 | become | ✘ | +| 6 | become_exe | ✘ | +| 7 | become_flags | ✘ | +| 8 | become_method | ✘ | +| 9 | become_user | ✘ | +| 10 | changed_when | ✘ | +| 11 | check_mode | ✘ | +| 12 | collections | ✘ | +| 13 | debugger | ✘ | +| 14 | delay | ✘ | +| 15 | delegate_facts | ✘ | +| 16 | delegate_to | ✘ | +| 17 | diff | ✘ | +| 18 | environment | ✘ | +| 19 | failed_when | ✔︎ | +| 20 | ignore_errors | ✔︎ | +| 21 | ignore_unreachable | ✘ | +| 22 | local_action | ✘ | +| 23 | loop | ✔︎ | +| 24 | loop_control | ✘ | +| 25 | module_defaults | ✘ | +| 26 | name | ✔︎ | +| 27 | no_log | ✘ | +| 28 | notify | ✘ | +| 29 | poll | ✘ | +| 30 | port | ✘ | +| 31 | register | ✔︎ | +| 32 | remote_user | ✘ | +| 33 | retries | ✘ | +| 34 | run_once | ✘ | +| 35 | tags | ✔︎ | +| 36 | throttle | ✘ | +| 37 | timeout | ✘ | +| 38 | until | ✘ | +| 39 | vars | ✔︎ | +| 40 | when | ✔︎ | +| 41 | with_ | ✔︎ | ++------+------------------------+------------+ +*/ diff --git a/api/project/v1/handler.go b/api/project/v1/handler.go new file mode 100644 index 000000000..84da4d443 --- /dev/null +++ b/api/project/v1/handler.go @@ -0,0 +1,24 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +// Handler defined in project. +type Handler struct { + //Task + + Listen []string `yaml:"listen,omitempty"` +} diff --git a/api/project/v1/loop.go b/api/project/v1/loop.go new file mode 100644 index 000000000..e180d297a --- /dev/null +++ b/api/project/v1/loop.go @@ -0,0 +1,27 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +// LoopControl defined in project. +type LoopControl struct { + LoopVar string `yaml:"loop_var,omitempty"` + IndexVar string `yaml:"index_var,omitempty"` + Label string `yaml:"label,omitempty"` + Pause float32 `yaml:"pause,omitempty"` + Extended bool `yaml:"extended,omitempty"` + ExtendedAllitems bool `yaml:"extended_allitems,omitempty"` +} diff --git a/api/project/v1/notifiable.go b/api/project/v1/notifiable.go new file mode 100644 index 000000000..ea59c0649 --- /dev/null +++ b/api/project/v1/notifiable.go @@ -0,0 +1,22 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +// Notifiable defined in project. +type Notifiable struct { + Notify string `yaml:"notify,omitempty"` +} diff --git a/api/project/v1/play.go b/api/project/v1/play.go new file mode 100644 index 000000000..f1c3f4102 --- /dev/null +++ b/api/project/v1/play.go @@ -0,0 +1,99 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "github.com/cockroachdb/errors" + "gopkg.in/yaml.v3" +) + +// Play defined in project. +type Play struct { + ImportPlaybook string `yaml:"import_playbook,omitempty"` + + Base `yaml:",inline"` + Taggable `yaml:",inline"` + CollectionSearch `yaml:",inline"` + + PlayHost PlayHost `yaml:"hosts,omitempty"` + + // Facts + GatherFacts bool `yaml:"gather_facts,omitempty"` + + // defaults to be deprecated, should be 'None' in future + //GatherSubset []GatherSubset + //GatherTimeout int + //FactPath string + + // Variable Attribute + VarsFiles []string `yaml:"vars_files,omitempty"` + //VarsPrompt []string `yaml:"vars_prompt,omitempty"` + + // Role Attributes + Roles []Role `yaml:"roles,omitempty"` + + // Block (Task) Lists Attributes + Handlers []Block `yaml:"handlers,omitempty"` + PreTasks []Block `yaml:"pre_tasks,omitempty"` + PostTasks []Block `yaml:"post_tasks,omitempty"` + Tasks []Block `yaml:"tasks,omitempty"` + + // Flag/Setting Attributes + ForceHandlers bool `yaml:"force_handlers,omitempty"` + MaxFailPercentage float32 `yaml:"percent,omitempty"` + Serial PlaySerial `yaml:"serial,omitempty"` + Strategy string `yaml:"strategy,omitempty"` + Order string `yaml:"order,omitempty"` +} + +// PlaySerial defined in project. +type PlaySerial struct { + Data []any +} + +// UnmarshalYAML yaml string to serial. +func (s *PlaySerial) UnmarshalYAML(node *yaml.Node) error { + switch node.Kind { + case yaml.ScalarNode: + s.Data = []any{node.Value} + + return nil + case yaml.SequenceNode: + return node.Decode(&s.Data) + default: + return errors.New("unsupported type, excepted any or array") + } +} + +// PlayHost defined in project. +type PlayHost struct { + Hosts []string +} + +// UnmarshalYAML yaml string to play +func (p *PlayHost) UnmarshalYAML(node *yaml.Node) error { + switch node.Kind { + case yaml.ScalarNode: + p.Hosts = []string{node.Value} + + return nil + case yaml.SequenceNode: + return node.Decode(&p.Hosts) + default: + return errors.New("unsupported type, excepted string or string array") + } +} diff --git a/api/project/v1/play_test.go b/api/project/v1/play_test.go new file mode 100644 index 000000000..8db403560 --- /dev/null +++ b/api/project/v1/play_test.go @@ -0,0 +1,78 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" +) + +func TestUnmarshalSerial(t *testing.T) { + testcases := []struct { + name string + content string + except []any + }{ + { + name: "test single string", + content: ` +host1`, + except: []any{ + "host1", + }, + }, + { + name: "test single number", + content: ` +1`, + except: []any{ + "1", + }, + }, + { + name: "test single percent", + content: ` +10%`, + except: []any{ + "10%", + }, + }, + { + name: "test multi value", + content: ` +- host1 +- 1 +- 10%`, + except: []any{ + "host1", + 1, + "10%", + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + var serial PlaySerial + err := yaml.Unmarshal([]byte(tc.content), &serial) + assert.NoError(t, err) + assert.Equal(t, tc.except, serial.Data) + }) + } +} diff --git a/api/project/v1/playbook.go b/api/project/v1/playbook.go new file mode 100644 index 000000000..8a344ae91 --- /dev/null +++ b/api/project/v1/playbook.go @@ -0,0 +1,66 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "strings" + + "github.com/cockroachdb/errors" +) + +// NOTE: +// To unmarshal into a specific field of a struct, the field name must be explicitly specified. +// Otherwise, the UnmarshalYAML method associated with the field’s struct will be treated as a method of the parent struct, rather than the field itself. + +// Playbook defined in project. +type Playbook struct { + Play []Play +} + +// Validate playbook. delete empty ImportPlaybook which has convert to play. +func (p *Playbook) Validate() error { + var newPlay = make([]Play, 0) + for _, play := range p.Play { + // import_playbook is a link, should be ignored. + if play.ImportPlaybook != "" { + continue + } + + if len(play.PlayHost.Hosts) == 0 { + return errors.New("playbook's hosts must not be empty") + } + newPlay = append(newPlay, play) + } + p.Play = newPlay + + return nil +} + +// IsTmplSyntax Check if the string conforms to the template syntax. +func IsTmplSyntax(s string) bool { + return strings.Contains(s, "{{") && strings.Contains(s, "}}") +} + +// ParseTmplSyntax wraps a string with template syntax delimiters "{{" and "}}" +// to make it a valid Go template expression +func ParseTmplSyntax(s string) string { + return "{{ " + s + "}}" +} + +func TrimTmplSyntax(s string) string { + return strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(s, "{{"), "}}")) +} diff --git a/api/project/v1/playbook_test.go b/api/project/v1/playbook_test.go new file mode 100644 index 000000000..8c4f5a481 --- /dev/null +++ b/api/project/v1/playbook_test.go @@ -0,0 +1,267 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" +) + +func TestValidate(t *testing.T) { + testcases := []struct { + name string + playbook Playbook + }{ + { + name: "host is empty", + playbook: Playbook{Play: []Play{ + { + Base: Base{ + Name: "test", + }, + }, + }}, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + assert.Error(t, tc.playbook.Validate()) + }) + } +} + +func TestUnmarshalYamlPlaybook(t *testing.T) { + testcases := []struct { + name string + data []byte + excepted []Play + }{ + { + name: "Unmarshal hosts with single value", + data: []byte(`--- +- name: test play + hosts: localhost +`), + excepted: []Play{ + { + Base: Base{Name: "test play"}, + PlayHost: PlayHost{[]string{"localhost"}}, + }, + }, + }, + { + name: "Unmarshal hosts with multiple value", + data: []byte(`--- +- name: test play + hosts: ["control-plane", "worker"] +`), + excepted: []Play{ + { + Base: Base{ + Name: "test play", + }, + PlayHost: PlayHost{[]string{"control-plane", "worker"}}, + }, + }, + }, + { + name: "Unmarshal role with single value", + data: []byte(`--- +- name: test play + hosts: localhost + roles: + - test +`), + excepted: []Play{ + { + Base: Base{Name: "test play"}, + PlayHost: PlayHost{ + []string{"localhost"}, + }, + Roles: []Role{ + { + RoleInfo: RoleInfo{ + Role: "test", + }, + }, + }, + }, + }, + }, + { + name: "Unmarshal role with map value", + data: []byte(`--- +- name: test play + hosts: localhost + roles: + - role: test +`), + excepted: []Play{ + { + Base: Base{Name: "test play"}, + PlayHost: PlayHost{ + []string{"localhost"}, + }, + Roles: []Role{ + { + RoleInfo: RoleInfo{ + Role: "test", + }, + }, + }, + }, + }, + }, + { + name: "Unmarshal when with single value", + data: []byte(`--- +- name: test play + hosts: localhost + roles: + - role: test + when: "true" +`), + excepted: []Play{ + { + Base: Base{Name: "test play"}, + PlayHost: PlayHost{ + []string{"localhost"}, + }, + Roles: []Role{ + { + RoleInfo: RoleInfo{ + Conditional: Conditional{When: When{Data: []string{"true"}}}, + Role: "test", + }, + }, + }, + }, + }, + }, + { + name: "Unmarshal when with multiple value", + data: []byte(`--- +- name: test play + hosts: localhost + roles: + - role: test + when: ["true","false"] +`), + excepted: []Play{ + { + Base: Base{Name: "test play"}, + PlayHost: PlayHost{ + []string{"localhost"}, + }, + Roles: []Role{ + { + RoleInfo: RoleInfo{ + Conditional: Conditional{When: When{Data: []string{"true", "false"}}}, + Role: "test", + }, + }, + }, + }, + }, + }, + { + name: "Unmarshal single level block", + data: []byte(`--- +- name: test play + hosts: localhost + tasks: + - name: test + custom-module: abc +`), + excepted: []Play{ + { + Base: Base{Name: "test play"}, + PlayHost: PlayHost{Hosts: []string{"localhost"}}, + Tasks: []Block{ + { + BlockBase: BlockBase{Base: Base{Name: "test"}}, + Task: Task{UnknownField: map[string]any{"custom-module": "abc"}}, + }, + }, + }, + }, + }, + { + name: "Unmarshal multi level block", + data: []byte(`--- +- name: test play + hosts: localhost + tasks: + - name: test + block: + - name: test | test + custom-module: abc +`), + excepted: []Play{ + { + Base: Base{Name: "test play"}, + PlayHost: PlayHost{Hosts: []string{"localhost"}}, + Tasks: []Block{ + { + BlockBase: BlockBase{Base: Base{Name: "test"}}, + BlockInfo: BlockInfo{ + Block: []Block{{ + BlockBase: BlockBase{Base: Base{Name: "test | test"}}, + Task: Task{UnknownField: map[string]any{"custom-module": "abc"}}, + }}, + }, + }, + }, + }, + }, + }, + { + name: "Unmarshal include_tasks block", + data: []byte(`--- +- name: test play + hosts: localhost + tasks: + - include_tasks: task.yaml +`), + excepted: []Play{ + { + Base: Base{Name: "test play"}, + PlayHost: PlayHost{Hosts: []string{"localhost"}}, + Tasks: []Block{ + { + IncludeTasks: "task.yaml", + }, + }, + }, + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + var pb Playbook + err := yaml.Unmarshal(tc.data, &pb.Play) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, tc.excepted, pb.Play) + }) + } +} diff --git a/api/project/v1/role.go b/api/project/v1/role.go new file mode 100644 index 000000000..68ad9c4d4 --- /dev/null +++ b/api/project/v1/role.go @@ -0,0 +1,53 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "gopkg.in/yaml.v3" +) + +// Role defined in project. +type Role struct { + RoleInfo +} + +// RoleInfo defined in project. +type RoleInfo struct { + Base `yaml:",inline"` + Conditional `yaml:",inline"` + Taggable `yaml:",inline"` + CollectionSearch `yaml:",inline"` + + RoleDependency []Role `yaml:"dependencies,omitempty"` + + // Role ref in playbook + Role string `yaml:"role,omitempty"` + + Block []Block +} + +// UnmarshalYAML yaml string to role. +func (r *Role) UnmarshalYAML(node *yaml.Node) error { + switch node.Kind { + case yaml.ScalarNode: + r.Role = node.Value + case yaml.MappingNode: + return node.Decode(&r.RoleInfo) + } + + return nil +} diff --git a/api/project/v1/taggable.go b/api/project/v1/taggable.go new file mode 100644 index 000000000..b5d5355ef --- /dev/null +++ b/api/project/v1/taggable.go @@ -0,0 +1,92 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import "slices" + +// the special tags +const ( + // AlwaysTag it always run + AlwaysTag = "always" + // NeverTag it never run + NeverTag = "never" + // AllTag represent all tags + AllTag = "all" + // TaggedTag represent which has tags + TaggedTag = "tagged" +) + +// Taggable if it should executor +type Taggable struct { + Tags []string `yaml:"tags,omitempty"` +} + +// IsEnabled check if the block should be executed +func (t Taggable) IsEnabled(onlyTags []string, skipTags []string) bool { + shouldRun := true + + if len(onlyTags) > 0 { + switch { + case slices.Contains(t.Tags, AlwaysTag): + shouldRun = true + case slices.Contains(onlyTags, AllTag) && !slices.Contains(t.Tags, NeverTag): + shouldRun = true + case slices.Contains(onlyTags, TaggedTag) && !slices.Contains(t.Tags, NeverTag): + shouldRun = true + case !isdisjoint(onlyTags, t.Tags): + shouldRun = true + default: + shouldRun = false + } + } + + if shouldRun && len(skipTags) > 0 { + switch { + case slices.Contains(skipTags, AllTag) && + (!slices.Contains(t.Tags, AlwaysTag) || !slices.Contains(skipTags, AlwaysTag)): + shouldRun = false + case !isdisjoint(skipTags, t.Tags): + shouldRun = false + case slices.Contains(skipTags, TaggedTag) && len(skipTags) > 0: + shouldRun = false + } + } + + return shouldRun +} + +// JoinTag the child block should inherit tag for parent block +func JoinTag(child, parent Taggable) Taggable { + for _, tag := range parent.Tags { + if !slices.Contains(child.Tags, tag) { + child.Tags = append(child.Tags, tag) + } + } + + return child +} + +// isdisjoint returns true if a and b have no elements in common. +func isdisjoint(a, b []string) bool { + for _, s := range a { + if slices.Contains(b, s) { + return false + } + } + + return true +} diff --git a/api/v1beta1/auth_types.go b/api/v1beta1/auth_types.go deleted file mode 100644 index 1264deb23..000000000 --- a/api/v1beta1/auth_types.go +++ /dev/null @@ -1,54 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1beta1 - -import ( - "time" -) - -// Auth contains the SSH authentication configuration for machines. -type Auth struct { - // User is the username for SSH authentication. - // +optional - User string `yaml:"user,omitempty" json:"user,omitempty"` - - // Password is the password for SSH authentication. - // +optional - Password string `yaml:"password,omitempty" json:"password,omitempty"` - - // Port is the port for SSH authentication. - // +optional - Port *int `yaml:"port,omitempty" json:"port,omitempty"` - - // PrivateKey is the value of the private key for SSH authentication. - // +optional - PrivateKey string `yaml:"privateKey,omitempty" json:"privateKey,omitempty"` - - // PrivateKeyFile is the path to the private key for SSH authentication. - // +optional - PrivateKeyPath string `yaml:"privateKeyPath,omitempty" json:"privateKeyPath,omitempty"` - - // Secret is the secret of the PrivateKey or Password for SSH authentication.It should in the same namespace as capkk. - // When Password is empty, replace it with data.password. - // When PrivateKey is empty, replace it with data.privateKey - // +optional - Secret string `yaml:"secret,omitempty" json:"secret,omitempty"` - - // Timeout is the timeout for establish an SSH connection. - // +optional - Timeout *time.Duration `yaml:"timeout,omitempty" json:"timeout,omitempty"` -} diff --git a/api/v1beta1/common_types.go b/api/v1beta1/common_types.go deleted file mode 100644 index 04310c2ed..000000000 --- a/api/v1beta1/common_types.go +++ /dev/null @@ -1,22 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1beta1 - -const ( - // KKClusterLabelName is the label set on KKMachines and KKInstances linked to a kkCluster. - KKClusterLabelName = "kkcluster.infrastructure.cluster.x-k8s.io/cluster-name" -) diff --git a/api/v1beta1/component_types.go b/api/v1beta1/component_types.go deleted file mode 100644 index 36bbdffa7..000000000 --- a/api/v1beta1/component_types.go +++ /dev/null @@ -1,66 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1beta1 - -// Component is optional configuration for modifying the FTP server. -type Component struct { - // ZONE is the zone of the KKCluster where can get the binaries. - // If you have problem to access https://storage.googleapis.com, you can set "zone: cn". - // +optional - ZONE string `json:"zone,omitempty"` - - // Host is the host to download the binaries. - // +optional - Host string `json:"host,omitempty"` - - // Overrides is a list of components download information that need to be overridden. - // +optional - Overrides []Override `json:"overrides,omitempty"` -} - -// Override is a component download information that need to be overridden. -type Override struct { - // ID is the component id name. e.g. kubeadm, kubelet, containerd, etc. - ID string `json:"id,omitempty"` - - // Arch is the component arch. e.g. amd64, arm64, etc. - Arch string `json:"arch,omitempty"` - - // Version is the component version. e.g. v1.21.1, v1.22.0, etc. - Version string `json:"version,omitempty"` - - // URL is the download url of the binaries. - URL string `json:"url,omitempty"` - - // Path defines the URL path, which is the string of information that comes after the top level domain name. - Path string `json:"path,omitempty"` - - // Checksum is the SHA256 checksum of the binary. - // +optional - Checksum Checksum `json:"checksum,omitempty"` -} - -// Checksum is the SHA256 checksum of the binary. -type Checksum struct { - // Value is the checksum string value. - // +optional - Value string `json:"value,omitempty"` - - // Path defines the URL path, which is the path of the checksum file. - // +optional - Path string `json:"path,omitempty"` -} diff --git a/api/v1beta1/condition_consts.go b/api/v1beta1/condition_consts.go deleted file mode 100644 index 728ec876b..000000000 --- a/api/v1beta1/condition_consts.go +++ /dev/null @@ -1,135 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1beta1 - -import ( - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" -) - -// KKCluster condition -const ( - // PrincipalPreparedCondition reports whether the principal is prepared. - PrincipalPreparedCondition clusterv1.ConditionType = "PrincipalPrepared" -) - -const ( - // HostReadyCondition reports whether the host is ready to be used. - HostReadyCondition clusterv1.ConditionType = "HostReadyCondition" -) - -const ( - // ExternalLoadBalancerReadyCondition reports on whether a control plane load balancer was successfully reconciled. - ExternalLoadBalancerReadyCondition clusterv1.ConditionType = "ExternalLoadBalancerReady" - - // WaitForDNSNameResolveReason used while waiting for DNS name to resolve. - WaitForDNSNameResolveReason = "WaitForDNSNameResolve" -) - -const ( - // CallKKInstanceInPlaceUpgradeCondition reports whether set up the InPlaceUpgradeVersionAnnotation annotation on all the KKInstance conditions. - CallKKInstanceInPlaceUpgradeCondition clusterv1.ConditionType = "CallKKInstanceInPlaceUpgrade" - - // KKInstanceObjectNotUpdatedReason used when the KKInstance is not updated. - KKInstanceObjectNotUpdatedReason = "KKInstanceObjectNotUpdated" - - // AllKKInstancesUpgradeCompletedCondition reports whether all the KKInstances are upgraded. - AllKKInstancesUpgradeCompletedCondition clusterv1.ConditionType = "AllKKInstancesUpgradeCompleted" - - // WaitingForKKInstancesUpgradeReason used when the KKCluster is waiting for all KKInstance to be upgrading completed. - WaitingForKKInstancesUpgradeReason = "WaitingForKKInstancesUpgrade" -) - -// KKMachine condition -const ( - // InstanceReadyCondition reports on current status of the SSH instance. Ready indicates the instance is in a Running state. - InstanceReadyCondition clusterv1.ConditionType = "InstanceReady" - - // InstanceNotFoundReason used when the instance couldn't be retrieved. - InstanceNotFoundReason = "InstanceNotFound" - // InstanceCleanedReason instance is in a Cleared state. - InstanceCleanedReason = "InstanceCleaned" - // InstanceNotReadyReason used when the instance is in a pending state. - InstanceNotReadyReason = "InstanceNotReady" - // InstanceInPlaceUpgradingReason used when the instance is in a InstanceStateInPlaceUpgrading state. - InstanceInPlaceUpgradingReason = "InstanceInPlaceUpgrading" - // InstanceBootstrapStartedReason set when the provisioning of an instance started. - InstanceBootstrapStartedReason = "InstanceBootstrapStarted" - // InstanceBootstrapFailedReason used for failures during instance provisioning. - InstanceBootstrapFailedReason = "InstanceBootstrapFailed" - // WaitingForClusterInfrastructureReason used when machine is waiting for cluster infrastructure to be ready before proceeding. - WaitingForClusterInfrastructureReason = "WaitingForClusterInfrastructure" - // WaitingForBootstrapDataReason used when machine is waiting for bootstrap data to be ready before proceeding. - WaitingForBootstrapDataReason = "WaitingForBootstrapData" -) - -// KKInstance condition -const ( - // KKInstanceBootstrappedCondition reports on current status of the instance. Ready indicates the instance is in a init Bootstrapped state. - KKInstanceBootstrappedCondition clusterv1.ConditionType = "InstanceBootstrapped" - // KKInstanceInitOSFailedReason used when the instance couldn't initialize os environment. - KKInstanceInitOSFailedReason = "InitOSFailed" -) - -const ( - // KKInstanceRepositoryReadyCondition reports on whether successful to use repository to install packages. - KKInstanceRepositoryReadyCondition clusterv1.ConditionType = "InstanceRepositoryReady" - // KKInstanceRepositoryFailedReason used when the instance couldn't use repository to install packages. - KKInstanceRepositoryFailedReason = "InstanceRepositoryFailed" -) - -const ( - // KKInstanceBinariesReadyCondition reports on whether successful to download binaries. - KKInstanceBinariesReadyCondition clusterv1.ConditionType = "InstanceBinariesReady" - // KKInstanceGetBinaryFailedReason used when the instance couldn't download binaries (or check existed binaries). - KKInstanceGetBinaryFailedReason = "GetBinaryFailed" -) - -const ( - // KKInstanceCRIReadyCondition reports on whether successful to download and install CRI. - KKInstanceCRIReadyCondition clusterv1.ConditionType = "InstanceCRIReady" - // KKInstanceInstallCRIFailedReason used when the instance couldn't download and install CRI. - KKInstanceInstallCRIFailedReason = "InstallCRIFailed" -) - -const ( - // KKInstanceProvisionedCondition reports on whether the instance is provisioned by cloud-init. - KKInstanceProvisionedCondition clusterv1.ConditionType = "InstanceProvisioned" - // KKInstanceRunCloudConfigFailedReason used when the instance couldn't be provisioned. - KKInstanceRunCloudConfigFailedReason = "RunCloudConfigFailed" -) - -const ( - // KKInstanceDeletingBootstrapCondition reports on whether the instance is deleting bootstrap data. - KKInstanceDeletingBootstrapCondition clusterv1.ConditionType = "InstanceDeletingBootstrapped" - // KKInstanceClearEnvironmentFailedReason used when the instance couldn't be deleting bootstrap data. - KKInstanceClearEnvironmentFailedReason = "ClearEnvironmentFailed" - - // CleaningReason (Severity=Info) documents a machine node being cleaned. - CleaningReason = "Cleaning" -) - -const ( - // KKInstanceInPlaceUpgradedCondition reports on whether the instance is in place upgrade succeed. - KKInstanceInPlaceUpgradedCondition clusterv1.ConditionType = "KKInstanceInPlaceUpgraded" -) - -const ( - // KKInstanceInPlaceUpgradeBinariesCondition reports on whether the instance is in place upgrade download binaries succeed. - KKInstanceInPlaceUpgradeBinariesCondition clusterv1.ConditionType = "KKInstanceInPlaceUpgradeBinaries" - // KKInstanceInPlaceGetBinaryFailedReason used when the instance couldn't download binaries (or check existed binaries). - KKInstanceInPlaceGetBinaryFailedReason = "KKInstanceInPlaceUpgradeGetBinaryFailed" -) diff --git a/api/v1beta1/containermanager_types.go b/api/v1beta1/containermanager_types.go deleted file mode 100644 index cdb2d1e4c..000000000 --- a/api/v1beta1/containermanager_types.go +++ /dev/null @@ -1,53 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1beta1 - -// Default values. -const ( - DockerType = "docker" - DefaultDockerVersion = "20.10.8" - DefaultCRIDockerdVersion = "0.2.6" - DefaultDockerCRISocket = "unix:///run/cri-dockerd.sock" - - ContainerdType = "containerd" - DefaultContainerdVersion = "1.6.4" - DefaultContainerdCRISocket = "unix:///var/run/containerd/containerd.sock" - - DefaultCrictlVersion = "v1.24.0" -) - -// ContainerManager defines the desired state of ContainerManager -type ContainerManager struct { - // CRISocket is used to connect an existing CRIClient. - // +optional - CRISocket string `json:"criSocket,omitempty"` - - // Type defines the type of ContainerManager. - // "docker", "containerd" - Type string `json:"type,omitempty"` - - // Version defines the version of ContainerManager. - Version string `json:"version,omitempty"` - - // CRICTLVersion defines the version of CRICTL. - CRICTLVersion string `json:"crictlVersion,omitempty"` - - // CRIDockerdVersion defines the version of cri-dockerd, available only when Type is docker. - // https://github.com/Mirantis/cri-dockerd - // +optional - CRIDockerdVersion string `json:"criDockerdVersion,omitempty"` -} diff --git a/api/v1beta1/doc.go b/api/v1beta1/doc.go deleted file mode 100644 index f8ef23a5f..000000000 --- a/api/v1beta1/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package v1beta1 contains the v1beta1 API implementation. -package v1beta1 diff --git a/api/v1beta1/groupversion_info.go b/api/v1beta1/groupversion_info.go deleted file mode 100644 index 0e411b4a3..000000000 --- a/api/v1beta1/groupversion_info.go +++ /dev/null @@ -1,36 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package v1beta1 contains API Schema definitions for the infrastructure v1beta1 API group -// +kubebuilder:object:generate=true -// +groupName=infrastructure.cluster.x-k8s.io -package v1beta1 - -import ( - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/scheme" -) - -var ( - // GroupVersion is group version used to register these objects - GroupVersion = schema.GroupVersion{Group: "infrastructure.cluster.x-k8s.io", Version: "v1beta1"} - - // SchemeBuilder is used to add go types to the GroupVersionKind scheme - SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} - - // AddToScheme adds the types in this group-version to the given scheme. - AddToScheme = SchemeBuilder.AddToScheme -) diff --git a/api/v1beta1/kkcluster_types.go b/api/v1beta1/kkcluster_types.go deleted file mode 100644 index ec7d936e0..000000000 --- a/api/v1beta1/kkcluster_types.go +++ /dev/null @@ -1,191 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/errors" -) - -const ( - // ClusterFinalizer allows ReconcileKKCluster to clean up KK resources associated with KKCluster before - // removing it from the apiserver. - ClusterFinalizer = "kkcluster.infrastructure.cluster.x-k8s.io" - - // KUBERNETES the Kubernetes distributions - KUBERNETES = "kubernetes" - // K3S the K3S distributions - K3S = "k3s" - - // InPlaceUpgradeVersionAnnotation is the annotation that stores the version of the cluster used for in-place upgrade. - InPlaceUpgradeVersionAnnotation = "kkcluster.infrastructure.cluster.x-k8s.io/in-place-upgrade-version" -) - -// KKClusterSpec defines the desired state of KKCluster -type KKClusterSpec struct { - // Distribution represents the Kubernetes distribution type of the cluster. - Distribution string `json:"distribution,omitempty"` - - // Nodes represents the information about the nodes available to the cluster - Nodes Nodes `json:"nodes"` - - // ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. - // +optional - ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"` - - // ControlPlaneLoadBalancer is optional configuration for customizing control plane behavior. - ControlPlaneLoadBalancer *KKLoadBalancerSpec `json:"controlPlaneLoadBalancer,omitempty"` - - // Component is optional configuration for modifying the FTP server - // that gets the necessary components for the cluster. - // +optional - Component *Component `json:"component,omitempty"` - - // Registry represents the cluster image registry used to pull the images. - // +optional - Registry Registry `json:"registry,omitempty"` -} - -// Nodes represents the information about the nodes available to the cluster -type Nodes struct { - // Auth is the SSH authentication information of all instance. It is a global auth configuration. - // +optional - Auth Auth `json:"auth"` - - // Instances defines all instance contained in kkcluster. - Instances []InstanceInfo `json:"instances"` -} - -// InstanceInfo defines the information about the instance. -type InstanceInfo struct { - // Name is the host name of the machine. - // +kubebuilder:validation:MinLength=1 - Name string `json:"name,omitempty"` - - // Address is the IP address of the machine. - Address string `json:"address,omitempty"` - - // InternalAddress is the internal IP address of the machine. - InternalAddress string `json:"internalAddress,omitempty"` - - // Roles is the role of the machine. - // +optional - Roles []Role `json:"roles,omitempty"` - - // Arch is the architecture of the machine. e.g. "amd64", "arm64". - // +optional - Arch string `json:"arch,omitempty"` - - // Auth is the SSH authentication information of this machine. It will override the global auth configuration. - // +optional - Auth Auth `json:"auth,omitempty"` -} - -// KKLoadBalancerSpec defines the desired state of an KK load balancer. -type KKLoadBalancerSpec struct { - // The hostname on which the API server is serving. - Host string `json:"host,omitempty"` -} - -// KKClusterStatus defines the observed state of KKCluster -type KKClusterStatus struct { - // +kubebuilder:default=false - Ready bool `json:"ready"` - // FailureReason will be set in the event that there is a terminal problem - // reconciling the Machine and will contain a succinct value suitable - // for machine interpretation. - // - // This field should not be set for transitive errors that a controller - // faces that are expected to be fixed automatically over - // time (like service outages), but instead indicate that something is - // fundamentally wrong with the Machine's spec or the configuration of - // the controller, and that manual intervention is required. Examples - // of terminal errors would be invalid combinations of settings in the - // spec, values that are unsupported by the controller, or the - // responsible controller itself being critically misconfigured. - // - // Any transient errors that occur during the reconciliation of Machines - // can be added as events to the Machine object and/or logged in the - // controller's output. - // +optional - FailureReason *errors.MachineStatusError `json:"failureReason,omitempty"` - - // FailureMessage will be set in the event that there is a terminal problem - // reconciling the Machine and will contain a more verbose string suitable - // for logging and human consumption. - // - // This field should not be set for transitive errors that a controller - // faces that are expected to be fixed automatically over - // time (like service outages), but instead indicate that something is - // fundamentally wrong with the Machine's spec or the configuration of - // the controller, and that manual intervention is required. Examples - // of terminal errors would be invalid combinations of settings in the - // spec, values that are unsupported by the controller, or the - // responsible controller itself being critically misconfigured. - // - // Any transient errors that occur during the reconciliation of Machines - // can be added as events to the Machine object and/or logged in the - // controller's output. - // +optional - FailureMessage *string `json:"failureMessage,omitempty"` - - // Conditions defines current service state of the KKMachine. - // +optional - Conditions clusterv1.Conditions `json:"conditions,omitempty"` -} - -// +kubebuilder:object:root=true -// +kubebuilder:resource:path=kkclusters,scope=Namespaced,categories=cluster-api,shortName=kkc -// +kubebuilder:storageversion -// +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="Cluster",type="string",JSONPath=".metadata.labels.cluster\\.x-k8s\\.io/cluster-name",description="Cluster to which this KKClusters belongs" -// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.ready",description="Cluster infrastructure is ready for SSH instances" -// +kubebuilder:printcolumn:name="Endpoint",type="string",JSONPath=".spec.controlPlaneEndpoint",description="API Endpoint",priority=1 -// +k8s:defaulter-gen=true - -// KKCluster is the Schema for the kkclusters API -type KKCluster struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec KKClusterSpec `json:"spec,omitempty"` - Status KKClusterStatus `json:"status,omitempty"` -} - -//+kubebuilder:object:root=true - -// KKClusterList contains a list of KKCluster -type KKClusterList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []KKCluster `json:"items"` -} - -// GetConditions returns the observations of the operational state of the KKCluster resource. -func (k *KKCluster) GetConditions() clusterv1.Conditions { - return k.Status.Conditions -} - -// SetConditions sets the underlying service state of the KKCluster to the predescribed clusterv1.Conditions. -func (k *KKCluster) SetConditions(conditions clusterv1.Conditions) { - k.Status.Conditions = conditions -} - -func init() { - SchemeBuilder.Register(&KKCluster{}, &KKClusterList{}) -} diff --git a/api/v1beta1/kkcluster_webhook.go b/api/v1beta1/kkcluster_webhook.go deleted file mode 100644 index 0af9fb58f..000000000 --- a/api/v1beta1/kkcluster_webhook.go +++ /dev/null @@ -1,247 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - "fmt" - "net" - "strings" - "time" - - mapset "github.com/deckarep/golang-set" - "github.com/google/go-cmp/cmp" - "github.com/pkg/errors" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/validation/field" - "k8s.io/apiserver/pkg/storage/names" - "sigs.k8s.io/cluster-api/util/version" - ctrl "sigs.k8s.io/controller-runtime" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/webhook" -) - -const ( - defaultSSHUser = "root" - defaultSSHPort = 22 - defaultSSHEstablishTimeout = 30 * time.Second -) - -// log is for logging in this package. -var kkclusterlog = logf.Log.WithName("kkcluster-resource") - -func (k *KKCluster) SetupWebhookWithManager(mgr ctrl.Manager) error { - return ctrl.NewWebhookManagedBy(mgr). - For(k). - Complete() -} - -//+kubebuilder:webhook:path=/mutate-infrastructure-cluster-x-k8s-io-v1beta1-kkcluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=kkclusters,verbs=create;update,versions=v1beta1,name=default.kkcluster.infrastructure.cluster.x-k8s.io,admissionReviewVersions=v1 - -var _ webhook.Defaulter = &KKCluster{} - -// Default implements webhook.Defaulter so a webhook will be registered for the type -func (k *KKCluster) Default() { - kkclusterlog.Info("default", "name", k.Name) - - defaultDistribution(&k.Spec) - defaultAuth(&k.Spec.Nodes.Auth) - defaultInstance(&k.Spec) - defaultInPlaceUpgradeAnnotation(k.GetAnnotations()) -} - -func defaultDistribution(spec *KKClusterSpec) { - if spec.Distribution == "" { - spec.Distribution = "kubernetes" - } - if spec.Distribution == "k8s" { - spec.Distribution = "kubernetes" - } -} - -func defaultAuth(auth *Auth) { - if auth.User == "" { - auth.User = defaultSSHUser - } - if auth.Port == nil { - p := defaultSSHPort - auth.Port = &p - } - if auth.Timeout == nil { - t := defaultSSHEstablishTimeout - auth.Timeout = &t - } -} - -func defaultInstance(spec *KKClusterSpec) { - for i := range spec.Nodes.Instances { - instance := &spec.Nodes.Instances[i] - if instance.Name == "" { - instance.Name = names.SimpleNameGenerator.GenerateName("kk-instance-") - } - if instance.InternalAddress == "" { - instance.InternalAddress = instance.Address - } - if instance.Arch == "" { - instance.Arch = "amd64" - } - if len(instance.Roles) == 0 { - instance.Roles = []Role{ControlPlane, Worker} - } - } -} - -func defaultInPlaceUpgradeAnnotation(annotation map[string]string) { - upgradeVersion, ok := annotation[InPlaceUpgradeVersionAnnotation] - if !ok { - return - } - - if !strings.HasPrefix(upgradeVersion, "v") { - annotation[InPlaceUpgradeVersionAnnotation] = "v" + upgradeVersion - } -} - -//+kubebuilder:webhook:path=/validate-infrastructure-cluster-x-k8s-io-v1beta1-kkcluster,mutating=false,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=kkclusters,verbs=create;update,versions=v1beta1,name=validation.kkcluster.infrastructure.cluster.x-k8s.io,admissionReviewVersions=v1 - -var _ webhook.Validator = &KKCluster{} - -// ValidateCreate implements webhook.Validator so a webhook will be registered for the type -func (k *KKCluster) ValidateCreate() error { - kkclusterlog.Info("validate create", "name", k.Name) - - var allErrs field.ErrorList - allErrs = append(allErrs, validateDistribution(k.Spec)...) - allErrs = append(allErrs, validateClusterNodes(k.Spec.Nodes)...) - allErrs = append(allErrs, validateLoadBalancer(k.Spec.ControlPlaneLoadBalancer)...) - - return aggregateObjErrors(k.GroupVersionKind().GroupKind(), k.Name, allErrs) -} - -// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type -func (k *KKCluster) ValidateUpdate(old runtime.Object) error { - kkclusterlog.Info("validate update", "name", k.Name) - - var allErrs field.ErrorList - oldC, ok := old.(*KKCluster) - if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected an KKCluster but got a %T", old)) - } - - newLoadBalancer := &KKLoadBalancerSpec{} - - if k.Spec.ControlPlaneLoadBalancer != nil { - newLoadBalancer = k.Spec.ControlPlaneLoadBalancer.DeepCopy() - } - - if oldC.Spec.ControlPlaneLoadBalancer != nil { - // If old scheme was not nil, the new scheme should be the same. - existingLoadBalancer := oldC.Spec.ControlPlaneLoadBalancer.DeepCopy() - if !cmp.Equal(newLoadBalancer, existingLoadBalancer) { - allErrs = append(allErrs, - field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer"), - k.Spec.ControlPlaneLoadBalancer, "field is immutable"), - ) - } - } - - allErrs = append(allErrs, validateClusterNodes(k.Spec.Nodes)...) - allErrs = append(allErrs, validateInPlaceUpgrade(k.GetAnnotations())...) - return aggregateObjErrors(k.GroupVersionKind().GroupKind(), k.Name, allErrs) -} - -// ValidateDelete implements webhook.Validator so a webhook will be registered for the type -func (k *KKCluster) ValidateDelete() error { - return nil -} - -func validateDistribution(spec KKClusterSpec) []*field.Error { - var errs field.ErrorList - path := field.NewPath("spec", "distribution") - switch spec.Distribution { - case K3S: - return errs - case KUBERNETES: - return errs - default: - errs = append(errs, field.NotSupported(path, spec.Distribution, []string{K3S, KUBERNETES})) - } - return errs -} - -func validateLoadBalancer(loadBalancer *KKLoadBalancerSpec) []*field.Error { - var errs field.ErrorList - path := field.NewPath("spec", "controlPlaneLoadBalancer") - if loadBalancer.Host == "" { - errs = append(errs, field.Required(path.Child("host"), "can't be empty")) - } - return errs -} - -func validateClusterNodes(nodes Nodes) []*field.Error { - var errs field.ErrorList - - if nodes.Auth.Password == "" && nodes.Auth.PrivateKey == "" && nodes.Auth.PrivateKeyPath == "" && nodes.Auth.Secret == "" { - errs = append(errs, field.Required(field.NewPath("spec", "nodes", "auth"), "password and privateKey can't both be empty")) - } - - nameSet := mapset.NewThreadUnsafeSet() - addrSet := mapset.NewThreadUnsafeSet() - internalAddrSet := mapset.NewThreadUnsafeSet() - for i := range nodes.Instances { - instance := nodes.Instances[i] - path := field.NewPath("spec", "nodes", fmt.Sprintf("instances[%d]", i)) - if strings.ToLower(instance.Name) != instance.Name { - errs = append(errs, - field.Forbidden(path.Child("name"), "instance name must be the lower case")) - } - if !nameSet.Add(instance.Name) { - errs = append(errs, field.Duplicate(path.Child("name"), instance.Name)) - } - - if net.ParseIP(instance.Address) == nil { - errs = append(errs, field.Invalid(path.Child("address"), instance.Address, "instance address is invalid")) - } - if !addrSet.Add(instance.Address) { - errs = append(errs, field.Duplicate(path.Child("address"), instance.Address)) - } - - if net.ParseIP(instance.InternalAddress) == nil { - errs = append(errs, field.Invalid(path.Child("internalAddress"), instance.InternalAddress, "instance internalAddress is invalid")) - } - if !internalAddrSet.Add(instance.InternalAddress) { - errs = append(errs, field.Duplicate(path.Child("internalAddress"), instance.InternalAddress)) - } - } - return errs -} - -func validateInPlaceUpgrade(newAnnotation map[string]string) []*field.Error { - var allErrs field.ErrorList - - if v, ok := newAnnotation[InPlaceUpgradeVersionAnnotation]; ok { - _, err := version.ParseMajorMinorPatch(v) - if err != nil { - allErrs = append(allErrs, - field.InternalError( - field.NewPath("metadata", "annotations"), - errors.Wrapf(err, "failed to parse in-place upgrade version: %s", v)), - ) - } - } - return allErrs -} diff --git a/api/v1beta1/kkclustertemplate_types.go b/api/v1beta1/kkclustertemplate_types.go deleted file mode 100644 index a20f12289..000000000 --- a/api/v1beta1/kkclustertemplate_types.go +++ /dev/null @@ -1,63 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" -) - -// KKClusterTemplateSpec defines the desired state of KKClusterTemplate -type KKClusterTemplateSpec struct { - Template KKClusterTemplateResource `json:"template"` -} - -// +kubebuilder:object:root=true -// +kubebuilder:resource:path=kkclustertemplates,scope=Namespaced,categories=cluster-api,shortName=kkct -// +kubebuilder:storageversion -// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of KKClusterTemplate" -// +k8s:defaulter-gen=true - -// KKClusterTemplate is the Schema for the kkclustertemplates API -type KKClusterTemplate struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec KKClusterTemplateSpec `json:"spec,omitempty"` -} - -//+kubebuilder:object:root=true - -// KKClusterTemplateList contains a list of KKClusterTemplate -type KKClusterTemplateList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []KKClusterTemplate `json:"items"` -} - -func init() { - SchemeBuilder.Register(&KKClusterTemplate{}, &KKClusterTemplateList{}) -} - -// KKClusterTemplateResource Standard object's metadata -type KKClusterTemplateResource struct { - // Standard object's metadata. - // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - // +optional - ObjectMeta clusterv1.ObjectMeta `json:"metadata,omitempty"` - Spec KKClusterSpec `json:"spec"` -} diff --git a/api/v1beta1/kkclustertemplate_webhook.go b/api/v1beta1/kkclustertemplate_webhook.go deleted file mode 100644 index ce3656023..000000000 --- a/api/v1beta1/kkclustertemplate_webhook.go +++ /dev/null @@ -1,82 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - "github.com/google/go-cmp/cmp" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/validation/field" - ctrl "sigs.k8s.io/controller-runtime" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/webhook" -) - -// log is for logging in this package. -var kkclustertemplatelog = logf.Log.WithName("kkclustertemplate-resource") - -// SetupWebhookWithManager sets up and registers the webhook with the manager. -func (r *KKClusterTemplate) SetupWebhookWithManager(mgr ctrl.Manager) error { - return ctrl.NewWebhookManagedBy(mgr). - For(r). - Complete() -} - -//+kubebuilder:webhook:path=/mutate-infrastructure-cluster-x-k8s-io-v1beta1-kkclustertemplate,mutating=true,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=kkclustertemplates,verbs=create;update,versions=v1beta1,name=default.kkclustertemplate.infrastructure.cluster.x-k8s.io,admissionReviewVersions=v1 - -var _ webhook.Defaulter = &KKClusterTemplate{} - -// Default implements webhook.Defaulter so a webhook will be registered for the type -func (r *KKClusterTemplate) Default() { - kkclustertemplatelog.Info("default", "name", r.Name) - - defaultDistribution(&r.Spec.Template.Spec) - defaultAuth(&r.Spec.Template.Spec.Nodes.Auth) - defaultInstance(&r.Spec.Template.Spec) -} - -//+kubebuilder:webhook:path=/validate-infrastructure-cluster-x-k8s-io-v1beta1-kkclustertemplate,mutating=false,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=kkclustertemplates,verbs=create;update,versions=v1beta1,name=validation.kkclustertemplate.infrastructure.cluster.x-k8s.io,admissionReviewVersions=v1 - -var _ webhook.Validator = &KKClusterTemplate{} - -// ValidateCreate implements webhook.Validator so a webhook will be registered for the type -func (r *KKClusterTemplate) ValidateCreate() error { - kkclustertemplatelog.Info("validate create", "name", r.Name) - - var allErrs field.ErrorList - allErrs = append(allErrs, validateDistribution(r.Spec.Template.Spec)...) - allErrs = append(allErrs, validateClusterNodes(r.Spec.Template.Spec.Nodes)...) - allErrs = append(allErrs, validateLoadBalancer(r.Spec.Template.Spec.ControlPlaneLoadBalancer)...) - - return aggregateObjErrors(r.GroupVersionKind().GroupKind(), r.Name, allErrs) -} - -// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type -func (r *KKClusterTemplate) ValidateUpdate(old runtime.Object) error { - kkclustertemplatelog.Info("validate update", "name", r.Name) - - oldC := old.(*KKClusterTemplate) - if !cmp.Equal(r.Spec, oldC.Spec) { - return apierrors.NewBadRequest("KKClusterTemplate.Spec is immutable") - } - return nil -} - -// ValidateDelete implements webhook.Validator so a webhook will be registered for the type -func (r *KKClusterTemplate) ValidateDelete() error { - return nil -} diff --git a/api/v1beta1/kkinstance_types.go b/api/v1beta1/kkinstance_types.go deleted file mode 100644 index b495482ba..000000000 --- a/api/v1beta1/kkinstance_types.go +++ /dev/null @@ -1,206 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/errors" -) - -const ( - // InstanceFinalizer allows ReconcileKKInstance to clean up KubeKey resources associated with KKInstance before - // removing it from the apiserver. - InstanceFinalizer = "kkinstance.infrastructure.cluster.x-k8s.io" -) - -// InstanceState describes the state of an KK instance. -type InstanceState string - -var ( - // InstanceStatePending is the string representing an instance in a pending state. - InstanceStatePending = InstanceState("pending") - - // InstanceStateBootstrapping is the string representing an instance in a bootstrapping state. - InstanceStateBootstrapping = InstanceState("bootstrapping") - - // InstanceStateBootstrapped = InstanceState("bootstrapped") - - // InstanceStateRunning is the string representing an instance in a running state. - InstanceStateRunning = InstanceState("running") - - // InstanceStateInPlaceUpgrading is the string representing an instance in an in-place-upgrading state. - InstanceStateInPlaceUpgrading = InstanceState("InPlaceUpgrading") - - // InstanceStateCleaning is the string representing an instance in a cleaning state. - InstanceStateCleaning = InstanceState("cleaning") - - // InstanceStateCleaned is the string representing an instance in a cleared state. - InstanceStateCleaned = InstanceState("cleaned") - - // InstanceRunningStates defines the set of states in which an SSH instance is - // running or going to be running soon. - InstanceRunningStates = sets.NewString( - string(InstanceStatePending), - string(InstanceStateRunning), - ) - - // InstanceKnownStates represents all known KKInstance states. - InstanceKnownStates = InstanceRunningStates.Union( - sets.NewString( - string(InstanceStateBootstrapping), - string(InstanceStateCleaning), - string(InstanceStateCleaned), - string(InstanceStateInPlaceUpgrading), - ), - ) -) - -// KKInstanceSpec defines the desired state of KKInstance -type KKInstanceSpec struct { - // Name is the host name of the machine. - // +kubebuilder:validation:MinLength=1 - Name string `json:"name,omitempty"` - - // Address is the IP address of the machine. - Address string `json:"address,omitempty"` - - // InternalAddress is the internal IP address of the machine. - InternalAddress string `json:"internalAddress,omitempty"` - - // Roles is the role of the machine. - // +optional - Roles []Role `json:"roles,omitempty"` - - // Arch is the architecture of the machine. e.g. "amd64", "arm64". - // +optional - Arch string `json:"arch,omitempty"` - - // Auth is the SSH authentication information of this machine. It will override the global auth configuration. - // +optional - Auth Auth `json:"auth,omitempty"` - - // ContainerManager is the container manager config of this machine. - // +optional - ContainerManager ContainerManager `json:"containerManager,omitempty"` - - // Repository is the repository config of this machine. - // +optional - Repository *Repository `json:"repository,omitempty"` -} - -// KKInstanceStatus defines the observed state of KKInstance -type KKInstanceStatus struct { - // The current state of the instance. - State InstanceState `json:"instanceState,omitempty"` - - // NodeRef will point to the corresponding Node if it exists. - // +optional - NodeRef *corev1.ObjectReference `json:"nodeRef,omitempty"` - - // NodeInfo is a set of ids/uuids to uniquely identify the node. - // More info: https://kubernetes.io/docs/concepts/nodes/node/#info - // +optional - NodeInfo *corev1.NodeSystemInfo `json:"nodeInfo,omitempty"` - - // FailureReason will be set in the event that there is a terminal problem - // reconciling the Machine and will contain a succinct value suitable - // for machine interpretation. - // - // This field should not be set for transitive errors that a controller - // faces that are expected to be fixed automatically over - // time (like service outages), but instead indicate that something is - // fundamentally wrong with the Machine's spec or the configuration of - // the controller, and that manual intervention is required. Examples - // of terminal errors would be invalid combinations of settings in the - // spec, values that are unsupported by the controller, or the - // responsible controller itself being critically misconfigured. - // - // Any transient errors that occur during the reconciliation of Machines - // can be added as events to the Machine object and/or logged in the - // controller's output. - // +optional - FailureReason *errors.MachineStatusError `json:"failureReason,omitempty"` - - // FailureMessage will be set in the event that there is a terminal problem - // reconciling the Machine and will contain a more verbose string suitable - // for logging and human consumption. - // - // This field should not be set for transitive errors that a controller - // faces that are expected to be fixed automatically over - // time (like service outages), but instead indicate that something is - // fundamentally wrong with the Machine's spec or the configuration of - // the controller, and that manual intervention is required. Examples - // of terminal errors would be invalid combinations of settings in the - // spec, values that are unsupported by the controller, or the - // responsible controller itself being critically misconfigured. - // - // Any transient errors that occur during the reconciliation of Machines - // can be added as events to the Machine object and/or logged in the - // controller's output. - // +optional - FailureMessage *string `json:"failureMessage,omitempty"` - - // Conditions defines current service state of the KKMachine. - // +optional - Conditions clusterv1.Conditions `json:"conditions,omitempty"` -} - -// +kubebuilder:object:root=true -// +kubebuilder:subresource:status -// +kubebuilder:resource:path=kkinstances,scope=Namespaced,categories=cluster-api,shortName=kki -// +kubebuilder:storageversion -// +kubebuilder:printcolumn:name="Hostname",type="string",JSONPath=".spec.name",description="kubekey instance hostname" -// +kubebuilder:printcolumn:name="Address",type="string",JSONPath=".spec.address",description="kubekey instance address" -// +kubebuilder:printcolumn:name="State",type="string",JSONPath=".status.instanceState",description="KKInstance state" -// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of KKInstance" -// +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".status.nodeInfo.kubeletVersion",description="Kubernetes version associated with this KKInstance" -// +k8s:defaulter-gen=true - -// KKInstance is the Schema for the kkinstances API -type KKInstance struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec KKInstanceSpec `json:"spec,omitempty"` - Status KKInstanceStatus `json:"status,omitempty"` -} - -// GetConditions returns the observations of the operational state of the KKMachine resource. -func (k *KKInstance) GetConditions() clusterv1.Conditions { - return k.Status.Conditions -} - -// SetConditions sets the underlying service state of the KKInstance to the predescribed clusterv1.Conditions. -func (k *KKInstance) SetConditions(conditions clusterv1.Conditions) { - k.Status.Conditions = conditions -} - -//+kubebuilder:object:root=true - -// KKInstanceList contains a list of KKInstance -type KKInstanceList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []KKInstance `json:"items"` -} - -func init() { - SchemeBuilder.Register(&KKInstance{}, &KKInstanceList{}) -} diff --git a/api/v1beta1/kkinstance_webhook.go b/api/v1beta1/kkinstance_webhook.go deleted file mode 100644 index f3d7b6898..000000000 --- a/api/v1beta1/kkinstance_webhook.go +++ /dev/null @@ -1,112 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/validation/field" - "sigs.k8s.io/cluster-api/util/version" - ctrl "sigs.k8s.io/controller-runtime" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/webhook" -) - -// log is for logging in this package. -var kkinstancelog = logf.Log.WithName("kkinstance-resource") - -func (k *KKInstance) SetupWebhookWithManager(mgr ctrl.Manager) error { - return ctrl.NewWebhookManagedBy(mgr). - For(k). - Complete() -} - -//+kubebuilder:webhook:path=/mutate-infrastructure-cluster-x-k8s-io-v1beta1-kkinstance,mutating=true,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=kkinstances,verbs=create;update,versions=v1beta1,name=default.kkinstance.infrastructure.cluster.x-k8s.io,admissionReviewVersions=v1 - -var _ webhook.Defaulter = &KKInstance{} - -// Default implements webhook.Defaulter so a webhook will be registered for the type -func (k *KKInstance) Default() { - kkinstancelog.Info("default", "name", k.Name) -} - -//+kubebuilder:webhook:path=/validate-infrastructure-cluster-x-k8s-io-v1beta1-kkinstance,mutating=false,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=kkinstances,verbs=create;update,versions=v1beta1,name=validation.kkinstance.infrastructure.cluster.x-k8s.io,admissionReviewVersions=v1 - -var _ webhook.Validator = &KKInstance{} - -// ValidateCreate implements webhook.Validator so a webhook will be registered for the type -func (k *KKInstance) ValidateCreate() error { - kkinstancelog.Info("validate create", "name", k.Name) - return nil -} - -// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type -func (k *KKInstance) ValidateUpdate(old runtime.Object) error { - kkinstancelog.Info("validate update", "name", k.Name) - - var allErrs field.ErrorList - if v, ok := k.GetAnnotations()[InPlaceUpgradeVersionAnnotation]; ok { - if k.Status.NodeInfo == nil { - allErrs = append(allErrs, - field.Invalid( - field.NewPath("status", "nodeInfo"), - k.Status.NodeInfo, - "nodeInfo is required for in-place upgrade")) - } - newSemverVersion, err := version.ParseMajorMinorPatch(v) - if err != nil { - allErrs = append(allErrs, - field.InternalError( - field.NewPath("metadata", "annotations"), - errors.Wrapf(err, "failed to parse in-place upgrade version: %s", v)), - ) - } - oldSemverVersion, err := version.ParseMajorMinorPatch(k.Status.NodeInfo.KubeletVersion) - if err != nil { - allErrs = append(allErrs, - field.InternalError( - field.NewPath("status", "nodeInfo", "kubeletVersion"), - errors.Wrapf(err, "failed to parse old version: %s", k.Status.NodeInfo.KubeletVersion)), - ) - } - - if newSemverVersion.Equals(oldSemverVersion) { - allErrs = append(allErrs, - field.Invalid( - field.NewPath("metadata", "annotations"), - v, - "new version must be different from old version"), - ) - } - - if !(newSemverVersion.GT(oldSemverVersion) && - newSemverVersion.Major == oldSemverVersion.Major && - (newSemverVersion.Minor == oldSemverVersion.Minor+1 || newSemverVersion.Minor == oldSemverVersion.Minor)) { - allErrs = append(allErrs, - field.Invalid(field.NewPath("metadata", "annotations"), - v, "Skipping MINOR versions when upgrading is unsupported.")) - } - } - - return aggregateObjErrors(k.GroupVersionKind().GroupKind(), k.Name, allErrs) -} - -// ValidateDelete implements webhook.Validator so a webhook will be registered for the type -func (k *KKInstance) ValidateDelete() error { - kkinstancelog.Info("validate delete", "name", k.Name) - return nil -} diff --git a/api/v1beta1/kkmachine_types.go b/api/v1beta1/kkmachine_types.go deleted file mode 100644 index aabb77286..000000000 --- a/api/v1beta1/kkmachine_types.go +++ /dev/null @@ -1,148 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/errors" -) - -const ( - // MachineFinalizer allows ReconcileKKMachine to clean up KubeKey resources associated with KKMachine before - // removing it from the apiserver. - MachineFinalizer = "kkmachine.infrastructure.cluster.x-k8s.io" -) - -// KKMachineSpec defines the desired state of KKMachine -type KKMachineSpec struct { - // ProviderID is the unique identifier as specified by the kubekey provider. - ProviderID *string `json:"providerID,omitempty"` - - // InstanceID is the name of the KKInstance. - InstanceID *string `json:"instanceID,omitempty"` - - // Roles is the role of the machine. - // +optional - Roles []Role `json:"roles"` - - // ContainerManager is the container manager config of this machine. - // +optional - ContainerManager ContainerManager `json:"containerManager"` - - // Repository is the repository config of this machine. - // +optional - Repository *Repository `json:"repository,omitempty"` -} - -// KKMachineStatus defines the observed state of KKMachine -type KKMachineStatus struct { - // Ready is true when the provider resource is ready. - // +optional - Ready bool `json:"ready"` - - // Addresses contains the KK instance associated addresses. - Addresses []clusterv1.MachineAddress `json:"addresses,omitempty"` - - // InstanceState is the state of the KK instance for this machine. - // +optional - InstanceState *InstanceState `json:"instanceState,omitempty"` - - // FailureReason will be set in the event that there is a terminal problem - // reconciling the Machine and will contain a succinct value suitable - // for machine interpretation. - // - // This field should not be set for transitive errors that a controller - // faces that are expected to be fixed automatically over - // time (like service outages), but instead indicate that something is - // fundamentally wrong with the Machine's spec or the configuration of - // the controller, and that manual intervention is required. Examples - // of terminal errors would be invalid combinations of settings in the - // spec, values that are unsupported by the controller, or the - // responsible controller itself being critically misconfigured. - // - // Any transient errors that occur during the reconciliation of Machines - // can be added as events to the Machine object and/or logged in the - // controller's output. - // +optional - FailureReason *errors.MachineStatusError `json:"failureReason,omitempty"` - - // FailureMessage will be set in the event that there is a terminal problem - // reconciling the Machine and will contain a more verbose string suitable - // for logging and human consumption. - // - // This field should not be set for transitive errors that a controller - // faces that are expected to be fixed automatically over - // time (like service outages), but instead indicate that something is - // fundamentally wrong with the Machine's spec or the configuration of - // the controller, and that manual intervention is required. Examples - // of terminal errors would be invalid combinations of settings in the - // spec, values that are unsupported by the controller, or the - // responsible controller itself being critically misconfigured. - // - // Any transient errors that occur during the reconciliation of Machines - // can be added as events to the Machine object and/or logged in the - // controller's output. - // +optional - FailureMessage *string `json:"failureMessage,omitempty"` - - // Conditions defines current service state of the KKMachine. - // +optional - Conditions clusterv1.Conditions `json:"conditions,omitempty"` -} - -// +kubebuilder:object:root=true -// +kubebuilder:resource:path=kkmachines,scope=Namespaced,categories=cluster-api,shortName=kkm -// +kubebuilder:storageversion -// +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="Cluster",type="string",JSONPath=".metadata.labels.cluster\\.x-k8s\\.io/cluster-name",description="Cluster to which this KKMachine belongs" -// +kubebuilder:printcolumn:name="Instance",type="string",JSONPath=".spec.instanceID",description="KKInstance name" -// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.ready",description="Machine ready status" -// +kubebuilder:printcolumn:name="Machine",type="string",JSONPath=".metadata.ownerReferences[?(@.kind==\"Machine\")].name",description="Machine object which owns with this KKMachine" -// +k8s:defaulter-gen=true - -// KKMachine is the Schema for the kkmachines API -type KKMachine struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec KKMachineSpec `json:"spec,omitempty"` - Status KKMachineStatus `json:"status,omitempty"` -} - -// GetConditions returns the observations of the operational state of the KKMachine resource. -func (k *KKMachine) GetConditions() clusterv1.Conditions { - return k.Status.Conditions -} - -// SetConditions sets the underlying service state of the KKMachine to the predescribed clusterv1.Conditions. -func (k *KKMachine) SetConditions(conditions clusterv1.Conditions) { - k.Status.Conditions = conditions -} - -//+kubebuilder:object:root=true - -// KKMachineList contains a list of KKMachine -type KKMachineList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []KKMachine `json:"items"` -} - -func init() { - SchemeBuilder.Register(&KKMachine{}, &KKMachineList{}) -} diff --git a/api/v1beta1/kkmachine_webhook.go b/api/v1beta1/kkmachine_webhook.go deleted file mode 100644 index ff9b6e240..000000000 --- a/api/v1beta1/kkmachine_webhook.go +++ /dev/null @@ -1,119 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - "strings" - - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/validation/field" - ctrl "sigs.k8s.io/controller-runtime" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/webhook" -) - -// log is for logging in this package. -var kkmachinelog = logf.Log.WithName("kkmachine-resource") - -func (k *KKMachine) SetupWebhookWithManager(mgr ctrl.Manager) error { - return ctrl.NewWebhookManagedBy(mgr). - For(k). - Complete() -} - -//+kubebuilder:webhook:path=/mutate-infrastructure-cluster-x-k8s-io-v1beta1-kkmachine,mutating=true,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=kkmachines,verbs=create;update,versions=v1beta1,name=default.kkmachine.infrastructure.cluster.x-k8s.io,admissionReviewVersions=v1 - -var _ webhook.Defaulter = &KKMachine{} - -// Default implements webhook.Defaulter so a webhook will be registered for the type -func (k *KKMachine) Default() { - kkmachinelog.Info("default", "name", k.Name) - - defaultContainerManager(&k.Spec) -} - -func defaultContainerManager(spec *KKMachineSpec) { - // Direct connection to the user-provided CRI socket - if spec.ContainerManager.CRISocket != "" { - return - } - - if spec.ContainerManager.Type == "" { - spec.ContainerManager.Type = ContainerdType - } - - switch spec.ContainerManager.Type { - case ContainerdType: - if spec.ContainerManager.Version == "" { - spec.ContainerManager.Version = DefaultContainerdVersion - } - spec.ContainerManager.CRISocket = DefaultContainerdCRISocket - case DockerType: - if spec.ContainerManager.Version == "" { - spec.ContainerManager.Version = DefaultDockerVersion - } - if spec.ContainerManager.CRIDockerdVersion == "" { - spec.ContainerManager.CRIDockerdVersion = DefaultCRIDockerdVersion - } - spec.ContainerManager.CRISocket = DefaultDockerCRISocket - } - - if spec.ContainerManager.CRICTLVersion == "" { - spec.ContainerManager.CRICTLVersion = DefaultCrictlVersion - } - - spec.ContainerManager.Version = strings.TrimPrefix(spec.ContainerManager.Version, "v") - - if !strings.HasPrefix(spec.ContainerManager.CRICTLVersion, "v") { - spec.ContainerManager.CRICTLVersion = "v" + spec.ContainerManager.CRICTLVersion - } -} - -//+kubebuilder:webhook:path=/validate-infrastructure-cluster-x-k8s-io-v1beta1-kkmachine,mutating=false,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=kkmachines,verbs=create;update,versions=v1beta1,name=validation.kkmachine.infrastructure.cluster.x-k8s.io,admissionReviewVersions=v1 - -var _ webhook.Validator = &KKMachine{} - -// ValidateCreate implements webhook.Validator so a webhook will be registered for the type -func (k *KKMachine) ValidateCreate() error { - kkmachinelog.Info("validate create", "name", k.Name) - - var allErrs field.ErrorList - allErrs = append(allErrs, validateRepository(k.Spec.Repository)...) - return aggregateObjErrors(k.GroupVersionKind().GroupKind(), k.Name, allErrs) -} - -// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type -func (k *KKMachine) ValidateUpdate(old runtime.Object) error { - kkmachinelog.Info("validate update", "name", k.Name) - var allErrs field.ErrorList - allErrs = append(allErrs, validateRepository(k.Spec.Repository)...) - return aggregateObjErrors(k.GroupVersionKind().GroupKind(), k.Name, allErrs) -} - -// ValidateDelete implements webhook.Validator so a webhook will be registered for the type -func (k *KKMachine) ValidateDelete() error { - kkmachinelog.Info("validate delete", "name", k.Name) - return nil -} - -func validateRepository(repo *Repository) field.ErrorList { //nolint:unparam - var allErrs field.ErrorList - if repo == nil { - return allErrs - } - return allErrs -} diff --git a/api/v1beta1/kkmachine_webhook_test.go b/api/v1beta1/kkmachine_webhook_test.go deleted file mode 100644 index aebbd8ed1..000000000 --- a/api/v1beta1/kkmachine_webhook_test.go +++ /dev/null @@ -1,61 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1beta1 - -import ( - "testing" - - . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - utildefaulting "sigs.k8s.io/cluster-api/util/defaulting" -) - -func TestKKMachine_Default(t *testing.T) { - g := NewWithT(t) - kkm := &KKMachine{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "foobar", - }, - } - - t.Run("for KKMachine", utildefaulting.DefaultValidateTest(kkm)) - kkm.Default() - - g.Expect(kkm.Spec.ContainerManager.Type).To(Equal(ContainerdType)) - g.Expect(kkm.Spec.ContainerManager.Version).To(Equal(DefaultContainerdVersion)) - g.Expect(kkm.Spec.ContainerManager.CRICTLVersion).To(Equal(DefaultCrictlVersion)) - - kkm2 := &KKMachine{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "foobar", - }, - Spec: KKMachineSpec{ - ContainerManager: ContainerManager{ - Type: ContainerdType, - Version: "v1.6.4", - CRICTLVersion: "1.24.0", - }, - }, - } - - t.Run("for KKMachine2", utildefaulting.DefaultValidateTest(kkm2)) - kkm2.Default() - - g.Expect(kkm2.Spec.ContainerManager.Type).To(Equal(ContainerdType)) - g.Expect(kkm2.Spec.ContainerManager.Version).To(Equal("1.6.4")) - g.Expect(kkm2.Spec.ContainerManager.CRICTLVersion).To(Equal("v1.24.0")) -} diff --git a/api/v1beta1/kkmachinetemplate_types.go b/api/v1beta1/kkmachinetemplate_types.go deleted file mode 100644 index 150dd8ae1..000000000 --- a/api/v1beta1/kkmachinetemplate_types.go +++ /dev/null @@ -1,76 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" -) - -// KKMachineTemplateStatus defines a status for an KKMachineTemplate. -type KKMachineTemplateStatus struct { - // Capacity defines the resource capacity for this machine. - // This value is used for autoscaling from zero operations as defined in: - // https://github.com/kubernetes-sigs/cluster-api/blob/main/docs/proposals/20210310-opt-in-autoscaling-from-zero.md - // +optional - Capacity corev1.ResourceList `json:"capacity,omitempty"` -} - -// KKMachineTemplateSpec defines the desired state of KKMachineTemplate -type KKMachineTemplateSpec struct { - Template KKMachineTemplateResource `json:"template"` -} - -// KKMachineTemplateResource describes the data needed to create am KKMachine from a template. -type KKMachineTemplateResource struct { - // Standard object's metadata. - // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - // +optional - ObjectMeta clusterv1.ObjectMeta `json:"metadata,omitempty"` - - // Spec is the specification of the desired behavior of the machine. - Spec KKMachineSpec `json:"spec"` -} - -// +kubebuilder:object:root=true -// +kubebuilder:resource:path=kkmachinetemplates,scope=Namespaced,categories=cluster-api,shortName=kkmt -// +kubebuilder:storageversion -// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of KKMachineTemplate" -// +k8s:defaulter-gen=true - -// KKMachineTemplate is the Schema for the kkmachinetemplates API -type KKMachineTemplate struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec KKMachineTemplateSpec `json:"spec,omitempty"` - Status KKMachineTemplateStatus `json:"status,omitempty"` -} - -//+kubebuilder:object:root=true - -// KKMachineTemplateList contains a list of KKMachineTemplate -type KKMachineTemplateList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []KKMachineTemplate `json:"items"` -} - -func init() { - SchemeBuilder.Register(&KKMachineTemplate{}, &KKMachineTemplateList{}) -} diff --git a/api/v1beta1/kkmachinetemplate_webhook.go b/api/v1beta1/kkmachinetemplate_webhook.go deleted file mode 100644 index ae9142859..000000000 --- a/api/v1beta1/kkmachinetemplate_webhook.go +++ /dev/null @@ -1,76 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/validation/field" - ctrl "sigs.k8s.io/controller-runtime" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/webhook" -) - -// log is for logging in this package. -var kkmachinetemplatelog = logf.Log.WithName("kkmachinetemplate-resource") - -func (k *KKMachineTemplate) SetupWebhookWithManager(mgr ctrl.Manager) error { - return ctrl.NewWebhookManagedBy(mgr). - For(k). - Complete() -} - -//+kubebuilder:webhook:path=/mutate-infrastructure-cluster-x-k8s-io-v1beta1-kkmachinetemplate,mutating=true,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=kkmachinetemplates,verbs=create;update,versions=v1beta1,name=default.kkmachinetemplate.infrastructure.cluster.x-k8s.io,admissionReviewVersions=v1 - -var _ webhook.Defaulter = &KKMachineTemplate{} - -// Default implements webhook.Defaulter so a webhook will be registered for the type -func (k *KKMachineTemplate) Default() { - kkmachinetemplatelog.Info("default", "name", k.Name) - - defaultContainerManager(&k.Spec.Template.Spec) -} - -//+kubebuilder:webhook:path=/validate-infrastructure-cluster-x-k8s-io-v1beta1-kkmachinetemplate,mutating=false,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=kkmachinetemplates,verbs=create;update,versions=v1beta1,name=validation.kkmachinetemplate.infrastructure.cluster.x-k8s.io,admissionReviewVersions=v1 - -var _ webhook.Validator = &KKMachineTemplate{} - -// ValidateCreate implements webhook.Validator so a webhook will be registered for the type -func (k *KKMachineTemplate) ValidateCreate() error { - kkmachinetemplatelog.Info("validate create", "name", k.Name) - - spec := k.Spec.Template.Spec - var allErrs field.ErrorList - allErrs = append(allErrs, validateRepository(spec.Repository)...) - return aggregateObjErrors(k.GroupVersionKind().GroupKind(), k.Name, allErrs) -} - -// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type -func (k *KKMachineTemplate) ValidateUpdate(old runtime.Object) error { - kkmachinetemplatelog.Info("validate update", "name", k.Name) - - spec := k.Spec.Template.Spec - var allErrs field.ErrorList - allErrs = append(allErrs, validateRepository(spec.Repository)...) - return aggregateObjErrors(k.GroupVersionKind().GroupKind(), k.Name, allErrs) -} - -// ValidateDelete implements webhook.Validator so a webhook will be registered for the type -func (k *KKMachineTemplate) ValidateDelete() error { - kkmachinetemplatelog.Info("validate delete", "name", k.Name) - - return nil -} diff --git a/api/v1beta1/registry_types.go b/api/v1beta1/registry_types.go deleted file mode 100644 index 801e616f6..000000000 --- a/api/v1beta1/registry_types.go +++ /dev/null @@ -1,68 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1beta1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// Registry is the configuration for a cluster registry -type Registry struct { - metav1.TypeMeta `json:",inline"` - - // PrivateRegistry defines the private registry address of ContainerManager. - PrivateRegistry string `json:"privateRegistry"` - - // InsecureRegistries defines the insecure registries of ContainerManager. - InsecureRegistries []string `json:"insecureRegistries,omitempty"` - - // RegistryMirrors defines the registry mirrors of this PrivateRegistry. - RegistryMirrors []string `json:"registryMirrors,omitempty"` - - // NamespaceOverride defines the namespace override of this PrivateRegistry. - NamespaceOverride string `json:"namespaceOverride"` - - // Auth defines the auth of this PrivateRegistry. - Auth RegistryAuth `json:"auth"` -} - -// RegistryAuth defines the auth of a registry -type RegistryAuth struct { - // Username defines the username of this PrivateRegistry. - Username string `json:"username"` - - // Password defines the password of this PrivateRegistry. - Password string `json:"password"` - - // InsecureSkipVerify allow contacting this PrivateRegistry over HTTPS with failed TLS verification. - InsecureSkipVerify bool `json:"insecureSkipVerify"` - - // PlainHTTP allow contacting this PrivateRegistry over HTTP. - PlainHTTP bool `json:"plainHTTP"` - - // CertsPath defines the path of the certs files of this PrivateRegistry. - CertsPath string `json:"certsPath"` - - // CAFile is an SSL Certificate Authority file used to secure etcd communication. - CAFile string `yaml:"caFile" json:"caFile,omitempty"` - // CertFile is an SSL certification file used to secure etcd communication. - CertFile string `yaml:"certFile" json:"certFile,omitempty"` - // KeyFile is an SSL key file used to secure etcd communication. - KeyFile string `yaml:"keyFile" json:"keyFile,omitempty"` -} diff --git a/api/v1beta1/repository_types.go b/api/v1beta1/repository_types.go deleted file mode 100644 index 92e4bcb86..000000000 --- a/api/v1beta1/repository_types.go +++ /dev/null @@ -1,42 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1beta1 - -// ISO type -const ( - NONE = "none" - AUTO = "auto" -) - -// Repository defines the repository of the instance. -type Repository struct { - // ISO specifies the ISO file name. There are 3 options: - // "": empty string means will not install the packages. - // "none": no ISO file will be used. And capkk will use the default repository to install the required packages. - // "auto": capkk will detect the ISO file automatically. Only support Ubuntu/Debian/CentOS. - // "xxx-20.04-debs-amd64.iso": use the specified name to get the ISO file name. - // +optional - ISO string `json:"iso,omitempty"` - - // Update will update the repository packages list and cache if it is true. - // +optional - Update bool `json:"update,omitempty"` - - // Packages is a list of packages to be installed. - // +optional - Packages []string `json:"packages,omitempty"` -} diff --git a/api/v1beta1/role_types.go b/api/v1beta1/role_types.go deleted file mode 100644 index 7c8e6439f..000000000 --- a/api/v1beta1/role_types.go +++ /dev/null @@ -1,27 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1beta1 - -// Role represents a role of a node. -type Role string - -// Internal roles. -const ( - ControlPlane Role = "control-plane" - Master Role = "master" - Worker Role = "worker" -) diff --git a/api/v1beta1/webhook.go b/api/v1beta1/webhook.go deleted file mode 100644 index 80f26b58c..000000000 --- a/api/v1beta1/webhook.go +++ /dev/null @@ -1,35 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1beta1 - -import ( - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/validation/field" -) - -func aggregateObjErrors(gk schema.GroupKind, name string, allErrs field.ErrorList) error { - if len(allErrs) == 0 { - return nil - } - - return apierrors.NewInvalid( - gk, - name, - allErrs, - ) -} diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go deleted file mode 100644 index f6dfd6332..000000000 --- a/api/v1beta1/zz_generated.deepcopy.go +++ /dev/null @@ -1,839 +0,0 @@ -//go:build !ignore_autogenerated -// +build !ignore_autogenerated - -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by controller-gen. DO NOT EDIT. - -package v1beta1 - -import ( - "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - apiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/errors" - timex "time" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Auth) DeepCopyInto(out *Auth) { - *out = *in - if in.Port != nil { - in, out := &in.Port, &out.Port - *out = new(int) - **out = **in - } - if in.Timeout != nil { - in, out := &in.Timeout, &out.Timeout - *out = new(timex.Duration) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Auth. -func (in *Auth) DeepCopy() *Auth { - if in == nil { - return nil - } - out := new(Auth) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Checksum) DeepCopyInto(out *Checksum) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Checksum. -func (in *Checksum) DeepCopy() *Checksum { - if in == nil { - return nil - } - out := new(Checksum) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Component) DeepCopyInto(out *Component) { - *out = *in - if in.Overrides != nil { - in, out := &in.Overrides, &out.Overrides - *out = make([]Override, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Component. -func (in *Component) DeepCopy() *Component { - if in == nil { - return nil - } - out := new(Component) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ContainerManager) DeepCopyInto(out *ContainerManager) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContainerManager. -func (in *ContainerManager) DeepCopy() *ContainerManager { - if in == nil { - return nil - } - out := new(ContainerManager) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *InstanceInfo) DeepCopyInto(out *InstanceInfo) { - *out = *in - if in.Roles != nil { - in, out := &in.Roles, &out.Roles - *out = make([]Role, len(*in)) - copy(*out, *in) - } - in.Auth.DeepCopyInto(&out.Auth) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InstanceInfo. -func (in *InstanceInfo) DeepCopy() *InstanceInfo { - if in == nil { - return nil - } - out := new(InstanceInfo) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKCluster) DeepCopyInto(out *KKCluster) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKCluster. -func (in *KKCluster) DeepCopy() *KKCluster { - if in == nil { - return nil - } - out := new(KKCluster) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *KKCluster) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKClusterList) DeepCopyInto(out *KKClusterList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]KKCluster, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKClusterList. -func (in *KKClusterList) DeepCopy() *KKClusterList { - if in == nil { - return nil - } - out := new(KKClusterList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *KKClusterList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKClusterSpec) DeepCopyInto(out *KKClusterSpec) { - *out = *in - in.Nodes.DeepCopyInto(&out.Nodes) - out.ControlPlaneEndpoint = in.ControlPlaneEndpoint - if in.ControlPlaneLoadBalancer != nil { - in, out := &in.ControlPlaneLoadBalancer, &out.ControlPlaneLoadBalancer - *out = new(KKLoadBalancerSpec) - **out = **in - } - if in.Component != nil { - in, out := &in.Component, &out.Component - *out = new(Component) - (*in).DeepCopyInto(*out) - } - in.Registry.DeepCopyInto(&out.Registry) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKClusterSpec. -func (in *KKClusterSpec) DeepCopy() *KKClusterSpec { - if in == nil { - return nil - } - out := new(KKClusterSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKClusterStatus) DeepCopyInto(out *KKClusterStatus) { - *out = *in - if in.FailureReason != nil { - in, out := &in.FailureReason, &out.FailureReason - *out = new(errors.MachineStatusError) - **out = **in - } - if in.FailureMessage != nil { - in, out := &in.FailureMessage, &out.FailureMessage - *out = new(string) - **out = **in - } - if in.Conditions != nil { - in, out := &in.Conditions, &out.Conditions - *out = make(apiv1beta1.Conditions, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKClusterStatus. -func (in *KKClusterStatus) DeepCopy() *KKClusterStatus { - if in == nil { - return nil - } - out := new(KKClusterStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKClusterTemplate) DeepCopyInto(out *KKClusterTemplate) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKClusterTemplate. -func (in *KKClusterTemplate) DeepCopy() *KKClusterTemplate { - if in == nil { - return nil - } - out := new(KKClusterTemplate) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *KKClusterTemplate) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKClusterTemplateList) DeepCopyInto(out *KKClusterTemplateList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]KKClusterTemplate, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKClusterTemplateList. -func (in *KKClusterTemplateList) DeepCopy() *KKClusterTemplateList { - if in == nil { - return nil - } - out := new(KKClusterTemplateList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *KKClusterTemplateList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKClusterTemplateResource) DeepCopyInto(out *KKClusterTemplateResource) { - *out = *in - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKClusterTemplateResource. -func (in *KKClusterTemplateResource) DeepCopy() *KKClusterTemplateResource { - if in == nil { - return nil - } - out := new(KKClusterTemplateResource) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKClusterTemplateSpec) DeepCopyInto(out *KKClusterTemplateSpec) { - *out = *in - in.Template.DeepCopyInto(&out.Template) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKClusterTemplateSpec. -func (in *KKClusterTemplateSpec) DeepCopy() *KKClusterTemplateSpec { - if in == nil { - return nil - } - out := new(KKClusterTemplateSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKInstance) DeepCopyInto(out *KKInstance) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKInstance. -func (in *KKInstance) DeepCopy() *KKInstance { - if in == nil { - return nil - } - out := new(KKInstance) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *KKInstance) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKInstanceList) DeepCopyInto(out *KKInstanceList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]KKInstance, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKInstanceList. -func (in *KKInstanceList) DeepCopy() *KKInstanceList { - if in == nil { - return nil - } - out := new(KKInstanceList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *KKInstanceList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKInstanceSpec) DeepCopyInto(out *KKInstanceSpec) { - *out = *in - if in.Roles != nil { - in, out := &in.Roles, &out.Roles - *out = make([]Role, len(*in)) - copy(*out, *in) - } - in.Auth.DeepCopyInto(&out.Auth) - out.ContainerManager = in.ContainerManager - if in.Repository != nil { - in, out := &in.Repository, &out.Repository - *out = new(Repository) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKInstanceSpec. -func (in *KKInstanceSpec) DeepCopy() *KKInstanceSpec { - if in == nil { - return nil - } - out := new(KKInstanceSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKInstanceStatus) DeepCopyInto(out *KKInstanceStatus) { - *out = *in - if in.NodeRef != nil { - in, out := &in.NodeRef, &out.NodeRef - *out = new(v1.ObjectReference) - **out = **in - } - if in.NodeInfo != nil { - in, out := &in.NodeInfo, &out.NodeInfo - *out = new(v1.NodeSystemInfo) - **out = **in - } - if in.FailureReason != nil { - in, out := &in.FailureReason, &out.FailureReason - *out = new(errors.MachineStatusError) - **out = **in - } - if in.FailureMessage != nil { - in, out := &in.FailureMessage, &out.FailureMessage - *out = new(string) - **out = **in - } - if in.Conditions != nil { - in, out := &in.Conditions, &out.Conditions - *out = make(apiv1beta1.Conditions, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKInstanceStatus. -func (in *KKInstanceStatus) DeepCopy() *KKInstanceStatus { - if in == nil { - return nil - } - out := new(KKInstanceStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKLoadBalancerSpec) DeepCopyInto(out *KKLoadBalancerSpec) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKLoadBalancerSpec. -func (in *KKLoadBalancerSpec) DeepCopy() *KKLoadBalancerSpec { - if in == nil { - return nil - } - out := new(KKLoadBalancerSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKMachine) DeepCopyInto(out *KKMachine) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachine. -func (in *KKMachine) DeepCopy() *KKMachine { - if in == nil { - return nil - } - out := new(KKMachine) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *KKMachine) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKMachineList) DeepCopyInto(out *KKMachineList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]KKMachine, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachineList. -func (in *KKMachineList) DeepCopy() *KKMachineList { - if in == nil { - return nil - } - out := new(KKMachineList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *KKMachineList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKMachineSpec) DeepCopyInto(out *KKMachineSpec) { - *out = *in - if in.ProviderID != nil { - in, out := &in.ProviderID, &out.ProviderID - *out = new(string) - **out = **in - } - if in.InstanceID != nil { - in, out := &in.InstanceID, &out.InstanceID - *out = new(string) - **out = **in - } - if in.Roles != nil { - in, out := &in.Roles, &out.Roles - *out = make([]Role, len(*in)) - copy(*out, *in) - } - out.ContainerManager = in.ContainerManager - if in.Repository != nil { - in, out := &in.Repository, &out.Repository - *out = new(Repository) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachineSpec. -func (in *KKMachineSpec) DeepCopy() *KKMachineSpec { - if in == nil { - return nil - } - out := new(KKMachineSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKMachineStatus) DeepCopyInto(out *KKMachineStatus) { - *out = *in - if in.Addresses != nil { - in, out := &in.Addresses, &out.Addresses - *out = make([]apiv1beta1.MachineAddress, len(*in)) - copy(*out, *in) - } - if in.InstanceState != nil { - in, out := &in.InstanceState, &out.InstanceState - *out = new(InstanceState) - **out = **in - } - if in.FailureReason != nil { - in, out := &in.FailureReason, &out.FailureReason - *out = new(errors.MachineStatusError) - **out = **in - } - if in.FailureMessage != nil { - in, out := &in.FailureMessage, &out.FailureMessage - *out = new(string) - **out = **in - } - if in.Conditions != nil { - in, out := &in.Conditions, &out.Conditions - *out = make(apiv1beta1.Conditions, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachineStatus. -func (in *KKMachineStatus) DeepCopy() *KKMachineStatus { - if in == nil { - return nil - } - out := new(KKMachineStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKMachineTemplate) DeepCopyInto(out *KKMachineTemplate) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachineTemplate. -func (in *KKMachineTemplate) DeepCopy() *KKMachineTemplate { - if in == nil { - return nil - } - out := new(KKMachineTemplate) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *KKMachineTemplate) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKMachineTemplateList) DeepCopyInto(out *KKMachineTemplateList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]KKMachineTemplate, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachineTemplateList. -func (in *KKMachineTemplateList) DeepCopy() *KKMachineTemplateList { - if in == nil { - return nil - } - out := new(KKMachineTemplateList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *KKMachineTemplateList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKMachineTemplateResource) DeepCopyInto(out *KKMachineTemplateResource) { - *out = *in - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachineTemplateResource. -func (in *KKMachineTemplateResource) DeepCopy() *KKMachineTemplateResource { - if in == nil { - return nil - } - out := new(KKMachineTemplateResource) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKMachineTemplateSpec) DeepCopyInto(out *KKMachineTemplateSpec) { - *out = *in - in.Template.DeepCopyInto(&out.Template) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachineTemplateSpec. -func (in *KKMachineTemplateSpec) DeepCopy() *KKMachineTemplateSpec { - if in == nil { - return nil - } - out := new(KKMachineTemplateSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKMachineTemplateStatus) DeepCopyInto(out *KKMachineTemplateStatus) { - *out = *in - if in.Capacity != nil { - in, out := &in.Capacity, &out.Capacity - *out = make(v1.ResourceList, len(*in)) - for key, val := range *in { - (*out)[key] = val.DeepCopy() - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachineTemplateStatus. -func (in *KKMachineTemplateStatus) DeepCopy() *KKMachineTemplateStatus { - if in == nil { - return nil - } - out := new(KKMachineTemplateStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Nodes) DeepCopyInto(out *Nodes) { - *out = *in - in.Auth.DeepCopyInto(&out.Auth) - if in.Instances != nil { - in, out := &in.Instances, &out.Instances - *out = make([]InstanceInfo, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Nodes. -func (in *Nodes) DeepCopy() *Nodes { - if in == nil { - return nil - } - out := new(Nodes) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Override) DeepCopyInto(out *Override) { - *out = *in - out.Checksum = in.Checksum -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Override. -func (in *Override) DeepCopy() *Override { - if in == nil { - return nil - } - out := new(Override) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Registry) DeepCopyInto(out *Registry) { - *out = *in - out.TypeMeta = in.TypeMeta - if in.InsecureRegistries != nil { - in, out := &in.InsecureRegistries, &out.InsecureRegistries - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.RegistryMirrors != nil { - in, out := &in.RegistryMirrors, &out.RegistryMirrors - *out = make([]string, len(*in)) - copy(*out, *in) - } - out.Auth = in.Auth -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Registry. -func (in *Registry) DeepCopy() *Registry { - if in == nil { - return nil - } - out := new(Registry) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *Registry) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RegistryAuth) DeepCopyInto(out *RegistryAuth) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RegistryAuth. -func (in *RegistryAuth) DeepCopy() *RegistryAuth { - if in == nil { - return nil - } - out := new(RegistryAuth) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Repository) DeepCopyInto(out *Repository) { - *out = *in - if in.Packages != nil { - in, out := &in.Packages, &out.Packages - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Repository. -func (in *Repository) DeepCopy() *Repository { - if in == nil { - return nil - } - out := new(Repository) - in.DeepCopyInto(out) - return out -} diff --git a/bootstrap/k3s/PROJECT b/bootstrap/k3s/PROJECT deleted file mode 100644 index f9bf80aa2..000000000 --- a/bootstrap/k3s/PROJECT +++ /dev/null @@ -1,35 +0,0 @@ -domain: cluster.x-k8s.io -layout: -- go.kubebuilder.io/v3 -plugins: - manifests.sdk.operatorframework.io/v2: {} - scorecard.sdk.operatorframework.io/v2: {} -projectName: cluster-api-bootstrap-provider-kubekey-k3s -repo: github.com/kubesphere/kubekey/bootstrap/k3s -resources: -- api: - crdVersion: v1 - namespaced: true - controller: true - domain: cluster.x-k8s.io - group: bootstrap - kind: K3sConfig - path: github.com/kubesphere/kubekey/bootstrap/k3s/api/v1beta1 - version: v1beta1 - webhooks: - defaulting: true - validation: true - webhookVersion: v1 -- api: - crdVersion: v1 - namespaced: true - domain: cluster.x-k8s.io - group: bootstrap - kind: K3sConfigTemplate - path: github.com/kubesphere/kubekey/bootstrap/k3s/api/v1beta1 - version: v1beta1 - webhooks: - defaulting: true - validation: true - webhookVersion: v1 -version: "3" diff --git a/bootstrap/k3s/api/v1beta1/groupversion_info.go b/bootstrap/k3s/api/v1beta1/groupversion_info.go deleted file mode 100644 index 3c3871748..000000000 --- a/bootstrap/k3s/api/v1beta1/groupversion_info.go +++ /dev/null @@ -1,36 +0,0 @@ -/* -Copyright 2022. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package v1beta1 contains API Schema definitions for the bootstrap v1beta1 API group -// +kubebuilder:object:generate=true -// +groupName=bootstrap.cluster.x-k8s.io -package v1beta1 - -import ( - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/scheme" -) - -var ( - // GroupVersion is group version used to register these objects - GroupVersion = schema.GroupVersion{Group: "bootstrap.cluster.x-k8s.io", Version: "v1beta1"} - - // SchemeBuilder is used to add go types to the GroupVersionKind scheme - SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} - - // AddToScheme adds the types in this group-version to the given scheme. - AddToScheme = SchemeBuilder.AddToScheme -) diff --git a/bootstrap/k3s/api/v1beta1/k3sconfig.go b/bootstrap/k3s/api/v1beta1/k3sconfig.go deleted file mode 100644 index 2917085b0..000000000 --- a/bootstrap/k3s/api/v1beta1/k3sconfig.go +++ /dev/null @@ -1,212 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1beta1 - -// ServerConfiguration defines the desired state of k3s server configuration. -type ServerConfiguration struct { - // Database is the database configuration. - Database Database `json:"database,omitempty"` - - // Listener is the listener configuration. - Listener Listener `json:"listener,omitempty"` - - // Networking is the networking configuration. - Networking Networking `json:"networking,omitempty"` - - // KubernetesComponents is the kubernetes components configuration. - KubernetesComponents KubernetesComponents `json:"kubernetesComponents,omitempty"` - - // KubernetesProcesses is the kubernetes processes configuration. - KubernetesProcesses KubernetesProcesses `json:"kubernetesProcesses,omitempty"` - - // Agent is the agent configuration. - Agent AgentConfiguration `json:"agent,omitempty"` -} - -// AgentConfiguration defines the desired state of k3s agent configuration. -type AgentConfiguration struct { - // Node defines the k3s agent node configuration. - Node AgentNode `json:"node,omitempty"` - - // Runtime defines the k3s agent runtime configuration. - Runtime AgentRuntime `json:"runtime,omitempty"` - - // Networking defines the k3s agent networking configuration. - Networking AgentNetworking `json:"networking,omitempty"` - - // KubernetesAgentProcesses defines the k3s agent kubernetes processes configuration. - KubernetesAgentProcesses KubernetesAgentProcesses `json:"kubernetesAgentProcesses,omitempty"` -} - -// Database defines the desired state of k3s database configuration. -type Database struct { - // DataStoreEndPoint specify etcd, Mysql, Postgres, or Sqlite (default) data source name. - DataStoreEndPoint string `json:"dataStoreEndPoint,omitempty"` - - // DataStoreCAFile TLS Certificate Authority file used to secure datastore backend communication. - DataStoreCAFile string `json:"dataStoreCAFile,omitempty"` - - // DataStoreCertFile TLS certification file used to secure datastore backend communication. - DataStoreCertFile string `json:"dataStoreCertFile,omitempty"` - - // DataStoreKeyFile TLS key file used to secure datastore backend communication. - DataStoreKeyFile string `json:"dataStoreKeyFile,omitempty"` - - // ClusterInit initialize a new cluster using embedded Etcd. - ClusterInit *bool `json:"clusterInit,omitempty"` -} - -// Cluster is the desired state of k3s cluster configuration. -type Cluster struct { - // Token shared secret used to join a server or agent to a cluster. - Token string `json:"token,omitempty"` - - // TokenFile file containing the cluster-secret/token. - TokenFile string `json:"tokenFile,omitempty"` - - // Server which server to connect to, used to join a cluster. - Server string `json:"server,omitempty"` -} - -// Listener defines the desired state of k3s listener configuration. -type Listener struct { - // BindAddress k3s bind address. - BindAddress string `json:"bindAddress,omitempty"` - - // HTTPSListenPort HTTPS listen port. - HTTPSListenPort int `json:"httpsListenPort,omitempty"` - - // AdvertiseAddress IP address that apiserver uses to advertise to members of the cluster. - AdvertiseAddress string `json:"advertiseAddress,omitempty"` - - // AdvertisePort Port that apiserver uses to advertise to members of the cluster (default: listen-port). - AdvertisePort int `json:"advertisePort,omitempty"` - - // TLSSan Add additional hostname or IP as a Subject Alternative Name in the TLS cert. - TLSSan string `json:"tlsSan,omitempty"` -} - -// Networking defines the desired state of k3s networking configuration. -type Networking struct { - // ClusterCIDR Network CIDR to use for pod IPs. - ClusterCIDR string `json:"clusterCIDR,omitempty"` - - // ServiceCIDR Network CIDR to use for services IPs. - ServiceCIDR string `json:"serviceCIDR,omitempty"` - - // ServiceNodePortRange Port range to reserve for services with NodePort visibility. - ServiceNodePortRange string `json:"serviceNodePortRange,omitempty"` - - // ClusterDNS cluster IP for coredns service. Should be in your service-cidr range. - ClusterDNS string `json:"clusterDNS,omitempty"` - - // ClusterDomain cluster Domain. - ClusterDomain string `json:"clusterDomain,omitempty"` - - // FlannelBackend One of ‘none’, ‘vxlan’, ‘ipsec’, ‘host-gw’, or ‘wireguard’. (default: vxlan) - FlannelBackend string `json:"flannelBackend,omitempty"` -} - -// AgentNode defines the desired state of k3s agent node configuration. -type AgentNode struct { - // NodeName k3s node name. - NodeName string `json:"nodeName,omitempty"` - - // NodeLabels registering and starting kubelet with set of labels. - NodeLabels []string `json:"nodeLabels,omitempty"` - - // NodeTaints registering and starting kubelet with set of taints. - NodeTaints []string `json:"nodeTaints,omitempty"` - - // SeLinux Enable SELinux in containerd - SeLinux bool `json:"seLinux,omitempty"` - - // LBServerPort - // Local port for supervisor client load-balancer. - // If the supervisor and apiserver are not colocated an additional port 1 less than this port - // will also be used for the apiserver client load-balancer. (default: 6444) - LBServerPort int `json:"lbServerPort,omitempty"` - - // DataDir Folder to hold state. - DataDir string `json:"dataDir,omitempty"` -} - -// AgentRuntime defines the desired state of k3s agent runtime configuration. -type AgentRuntime struct { - // ContainerRuntimeEndpoint Disable embedded containerd and use alternative CRI implementation. - ContainerRuntimeEndpoint string `json:"containerRuntimeEndpoint,omitempty"` - - // PauseImage Customized pause image for containerd or Docker sandbox. - PauseImage string `json:"pauseImage,omitempty"` - - // PrivateRegistry Path to a private registry configuration file. - PrivateRegistry string `json:"privateRegistry,omitempty"` -} - -// AgentNetworking defines the desired state of k3s agent networking configuration. -type AgentNetworking struct { - // NodeIP IP address to advertise for node. - NodeIP string `json:"nodeIP,omitempty"` - - // NodeExternalIP External IP address to advertise for node. - NodeExternalIP string `json:"nodeExternalIP,omitempty"` - - // ResolvConf Path to Kubelet resolv.conf file. - ResolvConf string `json:"resolvConf,omitempty"` -} - -// KubernetesComponents defines the desired state of k3s kubernetes components configuration. -type KubernetesComponents struct { - // Disable do not deploy packaged components and delete any deployed components - // (valid items: coredns, servicelb, traefik,local-storage, metrics-server). - Disable string `json:"disable,omitempty"` - - // DisableKubeProxy disable running kube-proxy. - DisableKubeProxy bool `json:"disableKubeProxy,omitempty"` - - // DisableNetworkPolicy disable k3s default network policy controller. - DisableNetworkPolicy bool `json:"disableNetworkPolicy,omitempty"` - - // DisableHelmController disable Helm controller. - DisableHelmController bool `json:"disableHelmController,omitempty"` -} - -// KubernetesProcesses defines the desired state of kubernetes processes configuration. -type KubernetesProcesses struct { - // KubeAPIServerArgs is a customized flag for kube-apiserver process - // +optional - KubeAPIServerArgs []string `json:"kubeAPIServerArg,omitempty"` - - // KubeControllerManagerArgs is a customized flag for kube-controller-manager process - // +optional - KubeControllerManagerArgs []string `json:"kubeControllerManagerArgs,omitempty"` - - // KubeSchedulerArgs is a customized flag for kube-scheduler process - // +optional - KubeSchedulerArgs []string `json:"kubeSchedulerArgs,omitempty"` -} - -// KubernetesAgentProcesses defines the desired state of kubernetes agent processes configuration. -type KubernetesAgentProcesses struct { - // KubeletArgs Customized flag for kubelet process - // +optional - KubeletArgs []string `json:"kubeletArgs,omitempty"` - - // KubeProxyArgs Customized flag for kube-proxy process - // +optional - KubeProxyArgs []string `json:"kubeProxyArgs,omitempty"` -} diff --git a/bootstrap/k3s/api/v1beta1/k3sconfig_types.go b/bootstrap/k3s/api/v1beta1/k3sconfig_types.go deleted file mode 100644 index 572accb81..000000000 --- a/bootstrap/k3s/api/v1beta1/k3sconfig_types.go +++ /dev/null @@ -1,120 +0,0 @@ -/* -Copyright 2022. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" -) - -// K3sConfigSpec defines the desired state of K3sConfig -type K3sConfigSpec struct { - // Files specifies extra files to be passed to user_data upon creation. - // +optional - Files []bootstrapv1.File `json:"files,omitempty"` - - // Cluster defines the k3s cluster Options. - Cluster *Cluster `json:"cluster,omitempty"` - - // ServerConfiguration defines the k3s server configuration. - // +optional - ServerConfiguration *ServerConfiguration `json:"serverConfiguration,omitempty"` - - // AgentConfiguration defines the k3s agent configuration. - // +optional - AgentConfiguration *AgentConfiguration `json:"agentConfiguration,omitempty"` - - // PreK3sCommands specifies extra commands to run before k3s setup runs - // +optional - PreK3sCommands []string `json:"preK3sCommands,omitempty"` - - // PostK3sCommands specifies extra commands to run after k3s setup runs - // +optional - PostK3sCommands []string `json:"postK3sCommands,omitempty"` - - // Version specifies the k3s version - // +optional - Version string `json:"version,omitempty"` -} - -// K3sConfigStatus defines the observed state of K3sConfig -type K3sConfigStatus struct { - // Ready indicates the BootstrapData field is ready to be consumed - Ready bool `json:"ready,omitempty"` - - BootstrapData []byte `json:"bootstrapData,omitempty"` - - // DataSecretName is the name of the secret that stores the bootstrap data script. - // +optional - DataSecretName *string `json:"dataSecretName,omitempty"` - - // FailureReason will be set on non-retryable errors - // +optional - FailureReason string `json:"failureReason,omitempty"` - - // FailureMessage will be set on non-retryable errors - // +optional - FailureMessage string `json:"failureMessage,omitempty"` - - // ObservedGeneration is the latest generation observed by the controller. - // +optional - ObservedGeneration int64 `json:"observedGeneration,omitempty"` - - // Conditions defines current service state of the K3sConfig. - // +optional - Conditions clusterv1.Conditions `json:"conditions,omitempty"` -} - -// +kubebuilder:object:root=true -// +kubebuilder:resource:path=k3sconfigs,scope=Namespaced,categories=cluster-api -// +kubebuilder:storageversion -// +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="Cluster",type="string",JSONPath=".metadata.labels['cluster\\.x-k8s\\.io/cluster-name']",description="Cluster" -// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of K3sConfig" - -// K3sConfig is the Schema for the k3sConfigs API -type K3sConfig struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec K3sConfigSpec `json:"spec,omitempty"` - Status K3sConfigStatus `json:"status,omitempty"` -} - -// GetConditions returns the set of conditions for this object. -func (c *K3sConfig) GetConditions() clusterv1.Conditions { - return c.Status.Conditions -} - -// SetConditions sets the conditions on this object. -func (c *K3sConfig) SetConditions(conditions clusterv1.Conditions) { - c.Status.Conditions = conditions -} - -//+kubebuilder:object:root=true - -// K3sConfigList contains a list of K3sConfig -type K3sConfigList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []K3sConfig `json:"items"` -} - -func init() { - SchemeBuilder.Register(&K3sConfig{}, &K3sConfigList{}) -} diff --git a/bootstrap/k3s/api/v1beta1/k3sconfig_webhook.go b/bootstrap/k3s/api/v1beta1/k3sconfig_webhook.go deleted file mode 100644 index 5d4b0948d..000000000 --- a/bootstrap/k3s/api/v1beta1/k3sconfig_webhook.go +++ /dev/null @@ -1,146 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/validation/field" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/webhook" -) - -var ( - conflictingFileSourceMsg = "only one of content or contentFrom may be specified for a single file" - missingSecretNameMsg = "secret file source must specify non-empty secret name" - missingSecretKeyMsg = "secret file source must specify non-empty secret key" - pathConflictMsg = "path property must be unique among all files" -) - -func (c *K3sConfig) SetupWebhookWithManager(mgr ctrl.Manager) error { - return ctrl.NewWebhookManagedBy(mgr). - For(c). - Complete() -} - -// +kubebuilder:webhook:verbs=create;update,path=/mutate-bootstrap-cluster-x-k8s-io-v1beta1-k3sconfig,mutating=true,failurePolicy=fail,sideEffects=None,groups=bootstrap.cluster.x-k8s.io,resources=k3sconfigs,versions=v1beta1,name=default.k3sconfig.bootstrap.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 - -var _ webhook.Defaulter = &K3sConfig{} - -// Default implements webhook.Defaulter so a webhook will be registered for the type -func (c *K3sConfig) Default() { - DefaultK3sConfigSpec(&c.Spec) -} - -// DefaultK3sConfigSpec defaults a K3sConfigSpec. -func DefaultK3sConfigSpec(c *K3sConfigSpec) { -} - -// +kubebuilder:webhook:verbs=create;update,path=/validate-bootstrap-cluster-x-k8s-io-v1beta1-k3sconfig,mutating=false,failurePolicy=fail,matchPolicy=Equivalent,groups=bootstrap.cluster.x-k8s.io,resources=k3sconfigs,versions=v1beta1,name=validation.k3sconfig.bootstrap.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 - -var _ webhook.Validator = &K3sConfig{} - -// ValidateCreate implements webhook.Validator so a webhook will be registered for the type -func (c *K3sConfig) ValidateCreate() error { - return c.Spec.validate(c.Name) -} - -// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type -func (c *K3sConfig) ValidateUpdate(old runtime.Object) error { - return c.Spec.validate(c.Name) -} - -// ValidateDelete implements webhook.Validator so a webhook will be registered for the type -func (c *K3sConfig) ValidateDelete() error { - return nil -} - -func (c *K3sConfigSpec) validate(name string) error { - allErrs := c.Validate(field.NewPath("spec")) - - if len(allErrs) == 0 { - return nil - } - - return apierrors.NewInvalid(GroupVersion.WithKind("K3sConfig").GroupKind(), name, allErrs) -} - -// Validate ensures the K3sConfigSpec is valid. -func (c *K3sConfigSpec) Validate(pathPrefix *field.Path) field.ErrorList { - var allErrs field.ErrorList - - allErrs = append(allErrs, c.validateFiles(pathPrefix)...) - - return allErrs -} - -func (c *K3sConfigSpec) validateFiles(pathPrefix *field.Path) field.ErrorList { - var allErrs field.ErrorList - - knownPaths := map[string]struct{}{} - - for i := range c.Files { - file := c.Files[i] - if file.Content != "" && file.ContentFrom != nil { - allErrs = append( - allErrs, - field.Invalid( - pathPrefix.Child("files").Index(i), - file, - conflictingFileSourceMsg, - ), - ) - } - // n.b.: if we ever add types besides Secret as a ContentFrom - // Source, we must add webhook validation here for one of the - // sources being non-nil. - if file.ContentFrom != nil { - if file.ContentFrom.Secret.Name == "" { - allErrs = append( - allErrs, - field.Required( - pathPrefix.Child("files").Index(i).Child("contentFrom", "secret", "name"), - missingSecretNameMsg, - ), - ) - } - if file.ContentFrom.Secret.Key == "" { - allErrs = append( - allErrs, - field.Required( - pathPrefix.Child("files").Index(i).Child("contentFrom", "secret", "key"), - missingSecretKeyMsg, - ), - ) - } - } - _, conflict := knownPaths[file.Path] - if conflict { - allErrs = append( - allErrs, - field.Invalid( - pathPrefix.Child("files").Index(i).Child("path"), - file, - pathConflictMsg, - ), - ) - } - knownPaths[file.Path] = struct{}{} - } - - return allErrs -} diff --git a/bootstrap/k3s/api/v1beta1/k3sconfigtemplate_types.go b/bootstrap/k3s/api/v1beta1/k3sconfigtemplate_types.go deleted file mode 100644 index dce24f209..000000000 --- a/bootstrap/k3s/api/v1beta1/k3sconfigtemplate_types.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// K3sConfigTemplateSpec defines the desired state of K3sConfigTemplate -type K3sConfigTemplateSpec struct { - Template K3sConfigTemplateResource `json:"template"` -} - -// K3sConfigTemplateResource defines the Template structure -type K3sConfigTemplateResource struct { - Spec K3sConfigSpec `json:"spec,omitempty"` -} - -// +kubebuilder:object:root=true -// +kubebuilder:resource:path=k3sconfigtemplates,scope=Namespaced,categories=cluster-api -// +kubebuilder:storageversion -// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of K3sConfigTemplate" - -// K3sConfigTemplate is the Schema for the k3sconfigtemplates API -type K3sConfigTemplate struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec K3sConfigTemplateSpec `json:"spec,omitempty"` -} - -//+kubebuilder:object:root=true - -// K3sConfigTemplateList contains a list of K3sConfigTemplate -type K3sConfigTemplateList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []K3sConfigTemplate `json:"items"` -} - -func init() { - SchemeBuilder.Register(&K3sConfigTemplate{}, &K3sConfigTemplateList{}) -} diff --git a/bootstrap/k3s/api/v1beta1/k3sconfigtemplate_webhook.go b/bootstrap/k3s/api/v1beta1/k3sconfigtemplate_webhook.go deleted file mode 100644 index 626334b47..000000000 --- a/bootstrap/k3s/api/v1beta1/k3sconfigtemplate_webhook.go +++ /dev/null @@ -1,71 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/validation/field" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/webhook" -) - -func (r *K3sConfigTemplate) SetupWebhookWithManager(mgr ctrl.Manager) error { - return ctrl.NewWebhookManagedBy(mgr). - For(r). - Complete() -} - -// +kubebuilder:webhook:verbs=create;update,path=/mutate-bootstrap-cluster-x-k8s-io-v1beta1-k3sconfigtemplate,mutating=true,failurePolicy=fail,groups=bootstrap.cluster.x-k8s.io,resources=k3sconfigtemplates,versions=v1beta1,name=default.k3sconfigtemplate.bootstrap.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 - -var _ webhook.Defaulter = &K3sConfigTemplate{} - -// Default implements webhook.Defaulter so a webhook will be registered for the type -func (r *K3sConfigTemplate) Default() { - DefaultK3sConfigSpec(&r.Spec.Template.Spec) -} - -// +kubebuilder:webhook:verbs=create;update,path=/validate-bootstrap-cluster-x-k8s-io-v1beta1-k3sconfigtemplate,mutating=false,failurePolicy=fail,matchPolicy=Equivalent,groups=bootstrap.cluster.x-k8s.io,resources=k3sconfigtemplates,versions=v1beta1,name=validation.k3sconfigtemplate.bootstrap.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 - -var _ webhook.Validator = &K3sConfigTemplate{} - -// ValidateCreate implements webhook.Validator so a webhook will be registered for the type -func (r *K3sConfigTemplate) ValidateCreate() error { - return r.Spec.validate(r.Name) -} - -// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type -func (r *K3sConfigTemplate) ValidateUpdate(old runtime.Object) error { - return r.Spec.validate(r.Name) -} - -// ValidateDelete implements webhook.Validator so a webhook will be registered for the type -func (r *K3sConfigTemplate) ValidateDelete() error { - return nil -} - -func (r *K3sConfigTemplateSpec) validate(name string) error { - var allErrs field.ErrorList - - allErrs = append(allErrs, r.Template.Spec.Validate(field.NewPath("spec", "template", "spec"))...) - - if len(allErrs) == 0 { - return nil - } - - return apierrors.NewInvalid(GroupVersion.WithKind("K3sConfigTemplate").GroupKind(), name, allErrs) -} diff --git a/bootstrap/k3s/api/v1beta1/zz_generated.deepcopy.go b/bootstrap/k3s/api/v1beta1/zz_generated.deepcopy.go deleted file mode 100644 index ab819fe2e..000000000 --- a/bootstrap/k3s/api/v1beta1/zz_generated.deepcopy.go +++ /dev/null @@ -1,486 +0,0 @@ -//go:build !ignore_autogenerated -// +build !ignore_autogenerated - -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by controller-gen. DO NOT EDIT. - -package v1beta1 - -import ( - "k8s.io/apimachinery/pkg/runtime" - cluster_apiapiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1" - apiv1beta1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AgentConfiguration) DeepCopyInto(out *AgentConfiguration) { - *out = *in - in.Node.DeepCopyInto(&out.Node) - out.Runtime = in.Runtime - out.Networking = in.Networking - in.KubernetesAgentProcesses.DeepCopyInto(&out.KubernetesAgentProcesses) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentConfiguration. -func (in *AgentConfiguration) DeepCopy() *AgentConfiguration { - if in == nil { - return nil - } - out := new(AgentConfiguration) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AgentNetworking) DeepCopyInto(out *AgentNetworking) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentNetworking. -func (in *AgentNetworking) DeepCopy() *AgentNetworking { - if in == nil { - return nil - } - out := new(AgentNetworking) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AgentNode) DeepCopyInto(out *AgentNode) { - *out = *in - if in.NodeLabels != nil { - in, out := &in.NodeLabels, &out.NodeLabels - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.NodeTaints != nil { - in, out := &in.NodeTaints, &out.NodeTaints - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentNode. -func (in *AgentNode) DeepCopy() *AgentNode { - if in == nil { - return nil - } - out := new(AgentNode) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AgentRuntime) DeepCopyInto(out *AgentRuntime) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentRuntime. -func (in *AgentRuntime) DeepCopy() *AgentRuntime { - if in == nil { - return nil - } - out := new(AgentRuntime) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Cluster) DeepCopyInto(out *Cluster) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Cluster. -func (in *Cluster) DeepCopy() *Cluster { - if in == nil { - return nil - } - out := new(Cluster) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Database) DeepCopyInto(out *Database) { - *out = *in - if in.ClusterInit != nil { - in, out := &in.ClusterInit, &out.ClusterInit - *out = new(bool) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Database. -func (in *Database) DeepCopy() *Database { - if in == nil { - return nil - } - out := new(Database) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sConfig) DeepCopyInto(out *K3sConfig) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sConfig. -func (in *K3sConfig) DeepCopy() *K3sConfig { - if in == nil { - return nil - } - out := new(K3sConfig) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *K3sConfig) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sConfigList) DeepCopyInto(out *K3sConfigList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]K3sConfig, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sConfigList. -func (in *K3sConfigList) DeepCopy() *K3sConfigList { - if in == nil { - return nil - } - out := new(K3sConfigList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *K3sConfigList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sConfigSpec) DeepCopyInto(out *K3sConfigSpec) { - *out = *in - if in.Files != nil { - in, out := &in.Files, &out.Files - *out = make([]apiv1beta1.File, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.Cluster != nil { - in, out := &in.Cluster, &out.Cluster - *out = new(Cluster) - **out = **in - } - if in.ServerConfiguration != nil { - in, out := &in.ServerConfiguration, &out.ServerConfiguration - *out = new(ServerConfiguration) - (*in).DeepCopyInto(*out) - } - if in.AgentConfiguration != nil { - in, out := &in.AgentConfiguration, &out.AgentConfiguration - *out = new(AgentConfiguration) - (*in).DeepCopyInto(*out) - } - if in.PreK3sCommands != nil { - in, out := &in.PreK3sCommands, &out.PreK3sCommands - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.PostK3sCommands != nil { - in, out := &in.PostK3sCommands, &out.PostK3sCommands - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sConfigSpec. -func (in *K3sConfigSpec) DeepCopy() *K3sConfigSpec { - if in == nil { - return nil - } - out := new(K3sConfigSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sConfigStatus) DeepCopyInto(out *K3sConfigStatus) { - *out = *in - if in.BootstrapData != nil { - in, out := &in.BootstrapData, &out.BootstrapData - *out = make([]byte, len(*in)) - copy(*out, *in) - } - if in.DataSecretName != nil { - in, out := &in.DataSecretName, &out.DataSecretName - *out = new(string) - **out = **in - } - if in.Conditions != nil { - in, out := &in.Conditions, &out.Conditions - *out = make(cluster_apiapiv1beta1.Conditions, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sConfigStatus. -func (in *K3sConfigStatus) DeepCopy() *K3sConfigStatus { - if in == nil { - return nil - } - out := new(K3sConfigStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sConfigTemplate) DeepCopyInto(out *K3sConfigTemplate) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sConfigTemplate. -func (in *K3sConfigTemplate) DeepCopy() *K3sConfigTemplate { - if in == nil { - return nil - } - out := new(K3sConfigTemplate) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *K3sConfigTemplate) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sConfigTemplateList) DeepCopyInto(out *K3sConfigTemplateList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]K3sConfigTemplate, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sConfigTemplateList. -func (in *K3sConfigTemplateList) DeepCopy() *K3sConfigTemplateList { - if in == nil { - return nil - } - out := new(K3sConfigTemplateList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *K3sConfigTemplateList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sConfigTemplateResource) DeepCopyInto(out *K3sConfigTemplateResource) { - *out = *in - in.Spec.DeepCopyInto(&out.Spec) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sConfigTemplateResource. -func (in *K3sConfigTemplateResource) DeepCopy() *K3sConfigTemplateResource { - if in == nil { - return nil - } - out := new(K3sConfigTemplateResource) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sConfigTemplateSpec) DeepCopyInto(out *K3sConfigTemplateSpec) { - *out = *in - in.Template.DeepCopyInto(&out.Template) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sConfigTemplateSpec. -func (in *K3sConfigTemplateSpec) DeepCopy() *K3sConfigTemplateSpec { - if in == nil { - return nil - } - out := new(K3sConfigTemplateSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KubernetesAgentProcesses) DeepCopyInto(out *KubernetesAgentProcesses) { - *out = *in - if in.KubeletArgs != nil { - in, out := &in.KubeletArgs, &out.KubeletArgs - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.KubeProxyArgs != nil { - in, out := &in.KubeProxyArgs, &out.KubeProxyArgs - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesAgentProcesses. -func (in *KubernetesAgentProcesses) DeepCopy() *KubernetesAgentProcesses { - if in == nil { - return nil - } - out := new(KubernetesAgentProcesses) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KubernetesComponents) DeepCopyInto(out *KubernetesComponents) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesComponents. -func (in *KubernetesComponents) DeepCopy() *KubernetesComponents { - if in == nil { - return nil - } - out := new(KubernetesComponents) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KubernetesProcesses) DeepCopyInto(out *KubernetesProcesses) { - *out = *in - if in.KubeAPIServerArgs != nil { - in, out := &in.KubeAPIServerArgs, &out.KubeAPIServerArgs - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.KubeControllerManagerArgs != nil { - in, out := &in.KubeControllerManagerArgs, &out.KubeControllerManagerArgs - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.KubeSchedulerArgs != nil { - in, out := &in.KubeSchedulerArgs, &out.KubeSchedulerArgs - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesProcesses. -func (in *KubernetesProcesses) DeepCopy() *KubernetesProcesses { - if in == nil { - return nil - } - out := new(KubernetesProcesses) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Listener) DeepCopyInto(out *Listener) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Listener. -func (in *Listener) DeepCopy() *Listener { - if in == nil { - return nil - } - out := new(Listener) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Networking) DeepCopyInto(out *Networking) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Networking. -func (in *Networking) DeepCopy() *Networking { - if in == nil { - return nil - } - out := new(Networking) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ServerConfiguration) DeepCopyInto(out *ServerConfiguration) { - *out = *in - in.Database.DeepCopyInto(&out.Database) - out.Listener = in.Listener - out.Networking = in.Networking - out.KubernetesComponents = in.KubernetesComponents - in.KubernetesProcesses.DeepCopyInto(&out.KubernetesProcesses) - in.Agent.DeepCopyInto(&out.Agent) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServerConfiguration. -func (in *ServerConfiguration) DeepCopy() *ServerConfiguration { - if in == nil { - return nil - } - out := new(ServerConfiguration) - in.DeepCopyInto(out) - return out -} diff --git a/bootstrap/k3s/config/certmanager/certificate.yaml b/bootstrap/k3s/config/certmanager/certificate.yaml deleted file mode 100644 index 0f6452902..000000000 --- a/bootstrap/k3s/config/certmanager/certificate.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager 0.11 check https://docs.cert-manager.io/en/latest/tasks/upgrading/index.html for breaking changes -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize - dnsNames: - - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc - - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: $(SERVICE_NAME)-cert # this secret will not be prefixed, since it's not managed by kustomize \ No newline at end of file diff --git a/bootstrap/k3s/config/certmanager/kustomization.yaml b/bootstrap/k3s/config/certmanager/kustomization.yaml deleted file mode 100644 index bebea5a59..000000000 --- a/bootstrap/k3s/config/certmanager/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -resources: -- certificate.yaml - -configurations: -- kustomizeconfig.yaml diff --git a/bootstrap/k3s/config/certmanager/kustomizeconfig.yaml b/bootstrap/k3s/config/certmanager/kustomizeconfig.yaml deleted file mode 100644 index 28a895a40..000000000 --- a/bootstrap/k3s/config/certmanager/kustomizeconfig.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# This configuration is for teaching kustomize how to update name ref and var substitution -nameReference: -- kind: Issuer - group: cert-manager.io - fieldSpecs: - - kind: Certificate - group: cert-manager.io - path: spec/issuerRef/name - -varReference: -- kind: Certificate - group: cert-manager.io - path: spec/commonName -- kind: Certificate - group: cert-manager.io - path: spec/dnsNames -- kind: Certificate - group: cert-manager.io - path: spec/secretName diff --git a/bootstrap/k3s/config/crd/bases/bootstrap.cluster.x-k8s.io_k3sconfigs.yaml b/bootstrap/k3s/config/crd/bases/bootstrap.cluster.x-k8s.io_k3sconfigs.yaml deleted file mode 100644 index bc58504e9..000000000 --- a/bootstrap/k3s/config/crd/bases/bootstrap.cluster.x-k8s.io_k3sconfigs.yaml +++ /dev/null @@ -1,496 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.1 - creationTimestamp: null - name: k3sconfigs.bootstrap.cluster.x-k8s.io -spec: - group: bootstrap.cluster.x-k8s.io - names: - categories: - - cluster-api - kind: K3sConfig - listKind: K3sConfigList - plural: k3sconfigs - singular: k3sconfig - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Cluster - jsonPath: .metadata.labels['cluster\.x-k8s\.io/cluster-name'] - name: Cluster - type: string - - description: Time duration since creation of K3sConfig - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: K3sConfig is the Schema for the k3sConfigs API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: K3sConfigSpec defines the desired state of K3sConfig - properties: - agentConfiguration: - description: AgentConfiguration defines the k3s agent configuration. - properties: - kubernetesAgentProcesses: - description: KubernetesAgentProcesses defines the k3s agent kubernetes - processes configuration. - properties: - kubeProxyArgs: - description: KubeProxyArgs Customized flag for kube-proxy - process - items: - type: string - type: array - kubeletArgs: - description: KubeletArgs Customized flag for kubelet process - items: - type: string - type: array - type: object - networking: - description: Networking defines the k3s agent networking configuration. - properties: - nodeExternalIP: - description: NodeExternalIP External IP address to advertise - for node. - type: string - nodeIP: - description: NodeIP IP address to advertise for node. - type: string - resolvConf: - description: ResolvConf Path to Kubelet resolv.conf file. - type: string - type: object - node: - description: Node defines the k3s agent node configuration. - properties: - dataDir: - description: DataDir Folder to hold state. - type: string - lbServerPort: - description: 'LBServerPort Local port for supervisor client - load-balancer. If the supervisor and apiserver are not colocated - an additional port 1 less than this port will also be used - for the apiserver client load-balancer. (default: 6444)' - type: integer - nodeLabels: - description: NodeLabels registering and starting kubelet with - set of labels. - items: - type: string - type: array - nodeName: - description: NodeName k3s node name. - type: string - nodeTaints: - description: NodeTaints registering and starting kubelet with - set of taints. - items: - type: string - type: array - seLinux: - description: SeLinux Enable SELinux in containerd - type: boolean - type: object - runtime: - description: Runtime defines the k3s agent runtime configuration. - properties: - containerRuntimeEndpoint: - description: ContainerRuntimeEndpoint Disable embedded containerd - and use alternative CRI implementation. - type: string - pauseImage: - description: PauseImage Customized pause image for containerd - or Docker sandbox. - type: string - privateRegistry: - description: PrivateRegistry Path to a private registry configuration - file. - type: string - type: object - type: object - cluster: - description: Cluster defines the k3s cluster Options. - properties: - server: - description: Server which server to connect to, used to join a - cluster. - type: string - token: - description: Token shared secret used to join a server or agent - to a cluster. - type: string - tokenFile: - description: TokenFile file containing the cluster-secret/token. - type: string - type: object - files: - description: Files specifies extra files to be passed to user_data - upon creation. - items: - description: File defines the input for generating write_files in - cloud-init. - properties: - append: - description: Append specifies whether to append Content to existing - file if Path exists. - type: boolean - content: - description: Content is the actual content of the file. - type: string - contentFrom: - description: ContentFrom is a referenced source of content to - populate the file. - properties: - secret: - description: Secret represents a secret that should populate - this file. - properties: - key: - description: Key is the key in the secret's data map - for this value. - type: string - name: - description: Name of the secret in the KubeadmBootstrapConfig's - namespace to use. - type: string - required: - - key - - name - type: object - required: - - secret - type: object - encoding: - description: Encoding specifies the encoding of the file contents. - enum: - - base64 - - gzip - - gzip+base64 - type: string - owner: - description: Owner specifies the ownership of the file, e.g. - "root:root". - type: string - path: - description: Path specifies the full path on disk where to store - the file. - type: string - permissions: - description: Permissions specifies the permissions to assign - to the file, e.g. "0640". - type: string - required: - - path - type: object - type: array - postK3sCommands: - description: PostK3sCommands specifies extra commands to run after - k3s setup runs - items: - type: string - type: array - preK3sCommands: - description: PreK3sCommands specifies extra commands to run before - k3s setup runs - items: - type: string - type: array - serverConfiguration: - description: ServerConfiguration defines the k3s server configuration. - properties: - agent: - description: Agent is the agent configuration. - properties: - kubernetesAgentProcesses: - description: KubernetesAgentProcesses defines the k3s agent - kubernetes processes configuration. - properties: - kubeProxyArgs: - description: KubeProxyArgs Customized flag for kube-proxy - process - items: - type: string - type: array - kubeletArgs: - description: KubeletArgs Customized flag for kubelet process - items: - type: string - type: array - type: object - networking: - description: Networking defines the k3s agent networking configuration. - properties: - nodeExternalIP: - description: NodeExternalIP External IP address to advertise - for node. - type: string - nodeIP: - description: NodeIP IP address to advertise for node. - type: string - resolvConf: - description: ResolvConf Path to Kubelet resolv.conf file. - type: string - type: object - node: - description: Node defines the k3s agent node configuration. - properties: - dataDir: - description: DataDir Folder to hold state. - type: string - lbServerPort: - description: 'LBServerPort Local port for supervisor client - load-balancer. If the supervisor and apiserver are not - colocated an additional port 1 less than this port will - also be used for the apiserver client load-balancer. - (default: 6444)' - type: integer - nodeLabels: - description: NodeLabels registering and starting kubelet - with set of labels. - items: - type: string - type: array - nodeName: - description: NodeName k3s node name. - type: string - nodeTaints: - description: NodeTaints registering and starting kubelet - with set of taints. - items: - type: string - type: array - seLinux: - description: SeLinux Enable SELinux in containerd - type: boolean - type: object - runtime: - description: Runtime defines the k3s agent runtime configuration. - properties: - containerRuntimeEndpoint: - description: ContainerRuntimeEndpoint Disable embedded - containerd and use alternative CRI implementation. - type: string - pauseImage: - description: PauseImage Customized pause image for containerd - or Docker sandbox. - type: string - privateRegistry: - description: PrivateRegistry Path to a private registry - configuration file. - type: string - type: object - type: object - database: - description: Database is the database configuration. - properties: - clusterInit: - description: ClusterInit initialize a new cluster using embedded - Etcd. - type: boolean - dataStoreCAFile: - description: DataStoreCAFile TLS Certificate Authority file - used to secure datastore backend communication. - type: string - dataStoreCertFile: - description: DataStoreCertFile TLS certification file used - to secure datastore backend communication. - type: string - dataStoreEndPoint: - description: DataStoreEndPoint specify etcd, Mysql, Postgres, - or Sqlite (default) data source name. - type: string - dataStoreKeyFile: - description: DataStoreKeyFile TLS key file used to secure - datastore backend communication. - type: string - type: object - kubernetesComponents: - description: KubernetesComponents is the kubernetes components - configuration. - properties: - disable: - description: 'Disable do not deploy packaged components and - delete any deployed components (valid items: coredns, servicelb, - traefik,local-storage, metrics-server).' - type: string - disableHelmController: - description: DisableHelmController disable Helm controller. - type: boolean - disableKubeProxy: - description: DisableKubeProxy disable running kube-proxy. - type: boolean - disableNetworkPolicy: - description: DisableNetworkPolicy disable k3s default network - policy controller. - type: boolean - type: object - kubernetesProcesses: - description: KubernetesProcesses is the kubernetes processes configuration. - properties: - kubeAPIServerArg: - description: KubeAPIServerArgs is a customized flag for kube-apiserver - process - items: - type: string - type: array - kubeControllerManagerArgs: - description: KubeControllerManagerArgs is a customized flag - for kube-controller-manager process - items: - type: string - type: array - kubeSchedulerArgs: - description: KubeSchedulerArgs is a customized flag for kube-scheduler - process - items: - type: string - type: array - type: object - listener: - description: Listener is the listener configuration. - properties: - advertiseAddress: - description: AdvertiseAddress IP address that apiserver uses - to advertise to members of the cluster. - type: string - advertisePort: - description: 'AdvertisePort Port that apiserver uses to advertise - to members of the cluster (default: listen-port).' - type: integer - bindAddress: - description: BindAddress k3s bind address. - type: string - httpsListenPort: - description: HTTPSListenPort HTTPS listen port. - type: integer - tlsSan: - description: TLSSan Add additional hostname or IP as a Subject - Alternative Name in the TLS cert. - type: string - type: object - networking: - description: Networking is the networking configuration. - properties: - clusterCIDR: - description: ClusterCIDR Network CIDR to use for pod IPs. - type: string - clusterDNS: - description: ClusterDNS cluster IP for coredns service. Should - be in your service-cidr range. - type: string - clusterDomain: - description: ClusterDomain cluster Domain. - type: string - flannelBackend: - description: 'FlannelBackend One of ‘none’, ‘vxlan’, ‘ipsec’, - ‘host-gw’, or ‘wireguard’. (default: vxlan)' - type: string - serviceCIDR: - description: ServiceCIDR Network CIDR to use for services - IPs. - type: string - serviceNodePortRange: - description: ServiceNodePortRange Port range to reserve for - services with NodePort visibility. - type: string - type: object - type: object - version: - description: Version specifies the k3s version - type: string - type: object - status: - description: K3sConfigStatus defines the observed state of K3sConfig - properties: - bootstrapData: - format: byte - type: string - conditions: - description: Conditions defines current service state of the K3sConfig. - items: - description: Condition defines an observation of a Cluster API resource - operational state. - properties: - lastTransitionTime: - description: Last time the condition transitioned from one status - to another. This should be when the underlying condition changed. - If that is not known, then using the time when the API field - changed is acceptable. - format: date-time - type: string - message: - description: A human readable message indicating details about - the transition. This field may be empty. - type: string - reason: - description: The reason for the condition's last transition - in CamelCase. The specific API may choose whether or not this - field is considered a guaranteed API. This field may not be - empty. - type: string - severity: - description: Severity provides an explicit classification of - Reason code, so the users or machines can immediately understand - the current situation and act accordingly. The Severity field - MUST be set only when Status=False. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of condition in CamelCase or in foo.example.com/CamelCase. - Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. - type: string - required: - - lastTransitionTime - - status - - type - type: object - type: array - dataSecretName: - description: DataSecretName is the name of the secret that stores - the bootstrap data script. - type: string - failureMessage: - description: FailureMessage will be set on non-retryable errors - type: string - failureReason: - description: FailureReason will be set on non-retryable errors - type: string - observedGeneration: - description: ObservedGeneration is the latest generation observed - by the controller. - format: int64 - type: integer - ready: - description: Ready indicates the BootstrapData field is ready to be - consumed - type: boolean - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/bootstrap/k3s/config/crd/bases/bootstrap.cluster.x-k8s.io_k3sconfigtemplates.yaml b/bootstrap/k3s/config/crd/bases/bootstrap.cluster.x-k8s.io_k3sconfigtemplates.yaml deleted file mode 100644 index 9dce789bd..000000000 --- a/bootstrap/k3s/config/crd/bases/bootstrap.cluster.x-k8s.io_k3sconfigtemplates.yaml +++ /dev/null @@ -1,445 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.1 - creationTimestamp: null - name: k3sconfigtemplates.bootstrap.cluster.x-k8s.io -spec: - group: bootstrap.cluster.x-k8s.io - names: - categories: - - cluster-api - kind: K3sConfigTemplate - listKind: K3sConfigTemplateList - plural: k3sconfigtemplates - singular: k3sconfigtemplate - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Time duration since creation of K3sConfigTemplate - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: K3sConfigTemplate is the Schema for the k3sconfigtemplates API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: K3sConfigTemplateSpec defines the desired state of K3sConfigTemplate - properties: - template: - description: K3sConfigTemplateResource defines the Template structure - properties: - spec: - description: K3sConfigSpec defines the desired state of K3sConfig - properties: - agentConfiguration: - description: AgentConfiguration defines the k3s agent configuration. - properties: - kubernetesAgentProcesses: - description: KubernetesAgentProcesses defines the k3s - agent kubernetes processes configuration. - properties: - kubeProxyArgs: - description: KubeProxyArgs Customized flag for kube-proxy - process - items: - type: string - type: array - kubeletArgs: - description: KubeletArgs Customized flag for kubelet - process - items: - type: string - type: array - type: object - networking: - description: Networking defines the k3s agent networking - configuration. - properties: - nodeExternalIP: - description: NodeExternalIP External IP address to - advertise for node. - type: string - nodeIP: - description: NodeIP IP address to advertise for node. - type: string - resolvConf: - description: ResolvConf Path to Kubelet resolv.conf - file. - type: string - type: object - node: - description: Node defines the k3s agent node configuration. - properties: - dataDir: - description: DataDir Folder to hold state. - type: string - lbServerPort: - description: 'LBServerPort Local port for supervisor - client load-balancer. If the supervisor and apiserver - are not colocated an additional port 1 less than - this port will also be used for the apiserver client - load-balancer. (default: 6444)' - type: integer - nodeLabels: - description: NodeLabels registering and starting kubelet - with set of labels. - items: - type: string - type: array - nodeName: - description: NodeName k3s node name. - type: string - nodeTaints: - description: NodeTaints registering and starting kubelet - with set of taints. - items: - type: string - type: array - seLinux: - description: SeLinux Enable SELinux in containerd - type: boolean - type: object - runtime: - description: Runtime defines the k3s agent runtime configuration. - properties: - containerRuntimeEndpoint: - description: ContainerRuntimeEndpoint Disable embedded - containerd and use alternative CRI implementation. - type: string - pauseImage: - description: PauseImage Customized pause image for - containerd or Docker sandbox. - type: string - privateRegistry: - description: PrivateRegistry Path to a private registry - configuration file. - type: string - type: object - type: object - cluster: - description: Cluster defines the k3s cluster Options. - properties: - server: - description: Server which server to connect to, used to - join a cluster. - type: string - token: - description: Token shared secret used to join a server - or agent to a cluster. - type: string - tokenFile: - description: TokenFile file containing the cluster-secret/token. - type: string - type: object - files: - description: Files specifies extra files to be passed to user_data - upon creation. - items: - description: File defines the input for generating write_files - in cloud-init. - properties: - append: - description: Append specifies whether to append Content - to existing file if Path exists. - type: boolean - content: - description: Content is the actual content of the file. - type: string - contentFrom: - description: ContentFrom is a referenced source of content - to populate the file. - properties: - secret: - description: Secret represents a secret that should - populate this file. - properties: - key: - description: Key is the key in the secret's - data map for this value. - type: string - name: - description: Name of the secret in the KubeadmBootstrapConfig's - namespace to use. - type: string - required: - - key - - name - type: object - required: - - secret - type: object - encoding: - description: Encoding specifies the encoding of the - file contents. - enum: - - base64 - - gzip - - gzip+base64 - type: string - owner: - description: Owner specifies the ownership of the file, - e.g. "root:root". - type: string - path: - description: Path specifies the full path on disk where - to store the file. - type: string - permissions: - description: Permissions specifies the permissions to - assign to the file, e.g. "0640". - type: string - required: - - path - type: object - type: array - postK3sCommands: - description: PostK3sCommands specifies extra commands to run - after k3s setup runs - items: - type: string - type: array - preK3sCommands: - description: PreK3sCommands specifies extra commands to run - before k3s setup runs - items: - type: string - type: array - serverConfiguration: - description: ServerConfiguration defines the k3s server configuration. - properties: - agent: - description: Agent is the agent configuration. - properties: - kubernetesAgentProcesses: - description: KubernetesAgentProcesses defines the - k3s agent kubernetes processes configuration. - properties: - kubeProxyArgs: - description: KubeProxyArgs Customized flag for - kube-proxy process - items: - type: string - type: array - kubeletArgs: - description: KubeletArgs Customized flag for kubelet - process - items: - type: string - type: array - type: object - networking: - description: Networking defines the k3s agent networking - configuration. - properties: - nodeExternalIP: - description: NodeExternalIP External IP address - to advertise for node. - type: string - nodeIP: - description: NodeIP IP address to advertise for - node. - type: string - resolvConf: - description: ResolvConf Path to Kubelet resolv.conf - file. - type: string - type: object - node: - description: Node defines the k3s agent node configuration. - properties: - dataDir: - description: DataDir Folder to hold state. - type: string - lbServerPort: - description: 'LBServerPort Local port for supervisor - client load-balancer. If the supervisor and - apiserver are not colocated an additional port - 1 less than this port will also be used for - the apiserver client load-balancer. (default: - 6444)' - type: integer - nodeLabels: - description: NodeLabels registering and starting - kubelet with set of labels. - items: - type: string - type: array - nodeName: - description: NodeName k3s node name. - type: string - nodeTaints: - description: NodeTaints registering and starting - kubelet with set of taints. - items: - type: string - type: array - seLinux: - description: SeLinux Enable SELinux in containerd - type: boolean - type: object - runtime: - description: Runtime defines the k3s agent runtime - configuration. - properties: - containerRuntimeEndpoint: - description: ContainerRuntimeEndpoint Disable - embedded containerd and use alternative CRI - implementation. - type: string - pauseImage: - description: PauseImage Customized pause image - for containerd or Docker sandbox. - type: string - privateRegistry: - description: PrivateRegistry Path to a private - registry configuration file. - type: string - type: object - type: object - database: - description: Database is the database configuration. - properties: - clusterInit: - description: ClusterInit initialize a new cluster - using embedded Etcd. - type: boolean - dataStoreCAFile: - description: DataStoreCAFile TLS Certificate Authority - file used to secure datastore backend communication. - type: string - dataStoreCertFile: - description: DataStoreCertFile TLS certification file - used to secure datastore backend communication. - type: string - dataStoreEndPoint: - description: DataStoreEndPoint specify etcd, Mysql, - Postgres, or Sqlite (default) data source name. - type: string - dataStoreKeyFile: - description: DataStoreKeyFile TLS key file used to - secure datastore backend communication. - type: string - type: object - kubernetesComponents: - description: KubernetesComponents is the kubernetes components - configuration. - properties: - disable: - description: 'Disable do not deploy packaged components - and delete any deployed components (valid items: - coredns, servicelb, traefik,local-storage, metrics-server).' - type: string - disableHelmController: - description: DisableHelmController disable Helm controller. - type: boolean - disableKubeProxy: - description: DisableKubeProxy disable running kube-proxy. - type: boolean - disableNetworkPolicy: - description: DisableNetworkPolicy disable k3s default - network policy controller. - type: boolean - type: object - kubernetesProcesses: - description: KubernetesProcesses is the kubernetes processes - configuration. - properties: - kubeAPIServerArg: - description: KubeAPIServerArgs is a customized flag - for kube-apiserver process - items: - type: string - type: array - kubeControllerManagerArgs: - description: KubeControllerManagerArgs is a customized - flag for kube-controller-manager process - items: - type: string - type: array - kubeSchedulerArgs: - description: KubeSchedulerArgs is a customized flag - for kube-scheduler process - items: - type: string - type: array - type: object - listener: - description: Listener is the listener configuration. - properties: - advertiseAddress: - description: AdvertiseAddress IP address that apiserver - uses to advertise to members of the cluster. - type: string - advertisePort: - description: 'AdvertisePort Port that apiserver uses - to advertise to members of the cluster (default: - listen-port).' - type: integer - bindAddress: - description: BindAddress k3s bind address. - type: string - httpsListenPort: - description: HTTPSListenPort HTTPS listen port. - type: integer - tlsSan: - description: TLSSan Add additional hostname or IP - as a Subject Alternative Name in the TLS cert. - type: string - type: object - networking: - description: Networking is the networking configuration. - properties: - clusterCIDR: - description: ClusterCIDR Network CIDR to use for pod - IPs. - type: string - clusterDNS: - description: ClusterDNS cluster IP for coredns service. - Should be in your service-cidr range. - type: string - clusterDomain: - description: ClusterDomain cluster Domain. - type: string - flannelBackend: - description: 'FlannelBackend One of ‘none’, ‘vxlan’, - ‘ipsec’, ‘host-gw’, or ‘wireguard’. (default: vxlan)' - type: string - serviceCIDR: - description: ServiceCIDR Network CIDR to use for services - IPs. - type: string - serviceNodePortRange: - description: ServiceNodePortRange Port range to reserve - for services with NodePort visibility. - type: string - type: object - type: object - version: - description: Version specifies the k3s version - type: string - type: object - type: object - required: - - template - type: object - type: object - served: true - storage: true - subresources: {} diff --git a/bootstrap/k3s/config/crd/kustomization.yaml b/bootstrap/k3s/config/crd/kustomization.yaml deleted file mode 100644 index 71b422e9d..000000000 --- a/bootstrap/k3s/config/crd/kustomization.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# This kustomization.yaml is not intended to be run by itself, -# since it depends on service name and namespace that are out of this kustomize package. -# It should be run by config/default -resources: -- bases/bootstrap.cluster.x-k8s.io_k3sconfigs.yaml -- bases/bootstrap.cluster.x-k8s.io_k3sconfigtemplates.yaml -#+kubebuilder:scaffold:crdkustomizeresource - -commonLabels: - cluster.x-k8s.io/v1beta1: v1beta1 - -patchesStrategicMerge: -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. -# patches here are for enabling the conversion webhook for each CRD -- patches/webhook_in_k3sconfigs.yaml -- patches/webhook_in_k3sconfigtemplates.yaml -#+kubebuilder:scaffold:crdkustomizewebhookpatch - -# [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. -# patches here are for enabling the CA injection for each CRD -- patches/cainjection_in_k3sconfigs.yaml -- patches/cainjection_in_k3sconfigtemplates.yaml -#+kubebuilder:scaffold:crdkustomizecainjectionpatch - -# the following config is for teaching kustomize how to do kustomization for CRDs. -configurations: -- kustomizeconfig.yaml diff --git a/bootstrap/k3s/config/crd/kustomizeconfig.yaml b/bootstrap/k3s/config/crd/kustomizeconfig.yaml deleted file mode 100644 index ec5c150a9..000000000 --- a/bootstrap/k3s/config/crd/kustomizeconfig.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# This file is for teaching kustomize how to substitute name and namespace reference in CRD -nameReference: -- kind: Service - version: v1 - fieldSpecs: - - kind: CustomResourceDefinition - version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/name - -namespace: -- kind: CustomResourceDefinition - version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/namespace - create: false - -varReference: -- path: metadata/annotations diff --git a/bootstrap/k3s/config/crd/patches/cainjection_in_k3sconfigs.yaml b/bootstrap/k3s/config/crd/patches/cainjection_in_k3sconfigs.yaml deleted file mode 100644 index 899737859..000000000 --- a/bootstrap/k3s/config/crd/patches/cainjection_in_k3sconfigs.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: k3sconfigs.bootstrap.cluster.x-k8s.io diff --git a/bootstrap/k3s/config/crd/patches/cainjection_in_k3sconfigtemplates.yaml b/bootstrap/k3s/config/crd/patches/cainjection_in_k3sconfigtemplates.yaml deleted file mode 100644 index 5722b7b37..000000000 --- a/bootstrap/k3s/config/crd/patches/cainjection_in_k3sconfigtemplates.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: k3sconfigtemplates.bootstrap.cluster.x-k8s.io diff --git a/bootstrap/k3s/config/crd/patches/webhook_in_k3sconfigs.yaml b/bootstrap/k3s/config/crd/patches/webhook_in_k3sconfigs.yaml deleted file mode 100644 index 75b66bfe8..000000000 --- a/bootstrap/k3s/config/crd/patches/webhook_in_k3sconfigs.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: k3sconfigs.bootstrap.cluster.x-k8s.io -spec: - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: ["v1", "v1beta1"] - clientConfig: - # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, - # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) - caBundle: Cg== - service: - namespace: system - name: webhook-service - path: /convert diff --git a/bootstrap/k3s/config/crd/patches/webhook_in_k3sconfigtemplates.yaml b/bootstrap/k3s/config/crd/patches/webhook_in_k3sconfigtemplates.yaml deleted file mode 100644 index 79e1942f0..000000000 --- a/bootstrap/k3s/config/crd/patches/webhook_in_k3sconfigtemplates.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: k3sconfigtemplates.bootstrap.cluster.x-k8s.io -spec: - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: ["v1", "v1beta1"] - clientConfig: - # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, - # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) - caBundle: Cg== - service: - namespace: system - name: webhook-service - path: /convert diff --git a/bootstrap/k3s/config/default/kustomization.yaml b/bootstrap/k3s/config/default/kustomization.yaml deleted file mode 100644 index 105a8e5f8..000000000 --- a/bootstrap/k3s/config/default/kustomization.yaml +++ /dev/null @@ -1,54 +0,0 @@ -namePrefix: capkk-k3s-bootstrap- -namespace: capkk-k3s-bootstrap-system - -commonLabels: - cluster.x-k8s.io/provider: "bootstrap-k3s" - -resources: - - namespace.yaml - -bases: - - ../rbac - - ../manager - - ../crd - - ../certmanager - - ../webhook - -patchesStrategicMerge: - # Provide customizable hook for make targets. - - manager_image_patch.yaml - - manager_pull_policy.yaml - # Enable webhook. - - manager_webhook_patch.yaml - # Inject certificate in the webhook definition. - - webhookcainjection_patch.yaml - -configurations: - - kustomizeconfig.yaml -vars: - - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR - objref: - kind: Certificate - group: cert-manager.io - version: v1 - name: serving-cert # this name should match the one in certificate.yaml - fieldref: - fieldpath: metadata.namespace - - name: CERTIFICATE_NAME - objref: - kind: Certificate - group: cert-manager.io - version: v1 - name: serving-cert # this name should match the one in certificate.yaml - - name: SERVICE_NAMESPACE # namespace of the service - objref: - kind: Service - version: v1 - name: webhook-service - fieldref: - fieldpath: metadata.namespace - - name: SERVICE_NAME - objref: - kind: Service - version: v1 - name: webhook-service diff --git a/bootstrap/k3s/config/default/kustomizeconfig.yaml b/bootstrap/k3s/config/default/kustomizeconfig.yaml deleted file mode 100644 index eb191e64d..000000000 --- a/bootstrap/k3s/config/default/kustomizeconfig.yaml +++ /dev/null @@ -1,4 +0,0 @@ -# This configuration is for teaching kustomize how to update name ref and var substitution -varReference: -- kind: Deployment - path: spec/template/spec/volumes/secret/secretName diff --git a/bootstrap/k3s/config/default/manager_image_patch.yaml b/bootstrap/k3s/config/default/manager_image_patch.yaml deleted file mode 100644 index c17636d83..000000000 --- a/bootstrap/k3s/config/default/manager_image_patch.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - image: docker.io/kubespheredev/k3s-bootstrap-controller:main - name: manager diff --git a/bootstrap/k3s/config/default/manager_pull_policy.yaml b/bootstrap/k3s/config/default/manager_pull_policy.yaml deleted file mode 100644 index 74a0879c6..000000000 --- a/bootstrap/k3s/config/default/manager_pull_policy.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: manager - imagePullPolicy: Always diff --git a/bootstrap/k3s/config/default/manager_webhook_patch.yaml b/bootstrap/k3s/config/default/manager_webhook_patch.yaml deleted file mode 100644 index bccef6d70..000000000 --- a/bootstrap/k3s/config/default/manager_webhook_patch.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: manager - ports: - - containerPort: 9443 - name: webhook-server - protocol: TCP - volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true - volumes: - - name: cert - secret: - secretName: $(SERVICE_NAME)-cert diff --git a/bootstrap/k3s/config/default/namespace.yaml b/bootstrap/k3s/config/default/namespace.yaml deleted file mode 100644 index 8b55c3cd8..000000000 --- a/bootstrap/k3s/config/default/namespace.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - labels: - control-plane: controller-manager - name: system diff --git a/bootstrap/k3s/config/default/webhookcainjection_patch.yaml b/bootstrap/k3s/config/default/webhookcainjection_patch.yaml deleted file mode 100644 index 04c08d027..000000000 --- a/bootstrap/k3s/config/default/webhookcainjection_patch.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: mutating-webhook-configuration - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validating-webhook-configuration - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) diff --git a/bootstrap/k3s/config/manager/kustomization.yaml b/bootstrap/k3s/config/manager/kustomization.yaml deleted file mode 100644 index 7394a6d05..000000000 --- a/bootstrap/k3s/config/manager/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: - - manager.yaml diff --git a/bootstrap/k3s/config/manager/manager.yaml b/bootstrap/k3s/config/manager/manager.yaml deleted file mode 100644 index 34ae98a94..000000000 --- a/bootstrap/k3s/config/manager/manager.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - control-plane: controller-manager -spec: - selector: - matchLabels: - control-plane: controller-manager - replicas: 1 - template: - metadata: - labels: - control-plane: controller-manager - spec: - containers: - - command: - - /manager - args: - - "--leader-elect" - - "--metrics-bind-addr=localhost:8080" - image: controller:latest - name: manager - ports: - - containerPort: 9440 - name: healthz - protocol: TCP - readinessProbe: - httpGet: - path: /readyz - port: healthz - livenessProbe: - httpGet: - path: /healthz - port: healthz - terminationGracePeriodSeconds: 10 - serviceAccountName: manager - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane diff --git a/bootstrap/k3s/config/rbac/kustomization.yaml b/bootstrap/k3s/config/rbac/kustomization.yaml deleted file mode 100644 index 922158614..000000000 --- a/bootstrap/k3s/config/rbac/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -resources: -- service_account.yaml -- role.yaml -- role_binding.yaml -- leader_election_role.yaml -- leader_election_role_binding.yaml diff --git a/bootstrap/k3s/config/rbac/leader_election_role.yaml b/bootstrap/k3s/config/rbac/leader_election_role.yaml deleted file mode 100644 index 4190ec805..000000000 --- a/bootstrap/k3s/config/rbac/leader_election_role.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# permissions to do leader election. -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: leader-election-role -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch diff --git a/bootstrap/k3s/config/rbac/leader_election_role_binding.yaml b/bootstrap/k3s/config/rbac/leader_election_role_binding.yaml deleted file mode 100644 index a73dfa95d..000000000 --- a/bootstrap/k3s/config/rbac/leader_election_role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: leader-election-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: leader-election-role -subjects: - - kind: ServiceAccount - name: manager - namespace: system diff --git a/bootstrap/k3s/config/rbac/role.yaml b/bootstrap/k3s/config/rbac/role.yaml deleted file mode 100644 index 9a5f95547..000000000 --- a/bootstrap/k3s/config/rbac/role.yaml +++ /dev/null @@ -1,49 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: manager-role -rules: -- apiGroups: - - "" - resources: - - configmaps - - events - - secrets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - bootstrap.cluster.x-k8s.io - resources: - - k3sconfigs - - k3sconfigs/finalizers - - k3sconfigs/status - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - cluster.x-k8s.io - resources: - - clusters - - clusters/status - - machinepools - - machinepools/status - - machines - - machines/status - - machinesets - verbs: - - get - - list - - watch diff --git a/bootstrap/k3s/config/rbac/role_binding.yaml b/bootstrap/k3s/config/rbac/role_binding.yaml deleted file mode 100644 index 3ffc9c2e1..000000000 --- a/bootstrap/k3s/config/rbac/role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: manager-role -subjects: - - kind: ServiceAccount - name: manager - namespace: system diff --git a/bootstrap/k3s/config/rbac/service_account.yaml b/bootstrap/k3s/config/rbac/service_account.yaml deleted file mode 100644 index 77f747b53..000000000 --- a/bootstrap/k3s/config/rbac/service_account.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: manager - namespace: system diff --git a/bootstrap/k3s/config/samples/bootstrap_v1beta1_k3sconfig.yaml b/bootstrap/k3s/config/samples/bootstrap_v1beta1_k3sconfig.yaml deleted file mode 100644 index f2b353bc1..000000000 --- a/bootstrap/k3s/config/samples/bootstrap_v1beta1_k3sconfig.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: K3sConfig -metadata: - name: k3sconfig-sample -spec: - # TODO(user): Add fields here diff --git a/bootstrap/k3s/config/samples/bootstrap_v1beta1_k3sconfigtemplate.yaml b/bootstrap/k3s/config/samples/bootstrap_v1beta1_k3sconfigtemplate.yaml deleted file mode 100644 index c6d495d4d..000000000 --- a/bootstrap/k3s/config/samples/bootstrap_v1beta1_k3sconfigtemplate.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: K3sConfigTemplate -metadata: - name: k3sconfigtemplate-sample -spec: - # TODO(user): Add fields here diff --git a/bootstrap/k3s/config/samples/kustomization.yaml b/bootstrap/k3s/config/samples/kustomization.yaml deleted file mode 100644 index c60238eaf..000000000 --- a/bootstrap/k3s/config/samples/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -## Append samples you want in your CSV to this file as resources ## -resources: -- bootstrap_v1beta1_k3sconfig.yaml -- bootstrap_v1beta1_k3sconfigtemplate.yaml -#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/bootstrap/k3s/config/scorecard/bases/config.yaml b/bootstrap/k3s/config/scorecard/bases/config.yaml deleted file mode 100644 index c77047841..000000000 --- a/bootstrap/k3s/config/scorecard/bases/config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: scorecard.operatorframework.io/v1alpha3 -kind: Configuration -metadata: - name: config -stages: -- parallel: true - tests: [] diff --git a/bootstrap/k3s/config/scorecard/kustomization.yaml b/bootstrap/k3s/config/scorecard/kustomization.yaml deleted file mode 100644 index 50cd2d084..000000000 --- a/bootstrap/k3s/config/scorecard/kustomization.yaml +++ /dev/null @@ -1,16 +0,0 @@ -resources: -- bases/config.yaml -patchesJson6902: -- path: patches/basic.config.yaml - target: - group: scorecard.operatorframework.io - version: v1alpha3 - kind: Configuration - name: config -- path: patches/olm.config.yaml - target: - group: scorecard.operatorframework.io - version: v1alpha3 - kind: Configuration - name: config -#+kubebuilder:scaffold:patchesJson6902 diff --git a/bootstrap/k3s/config/scorecard/patches/basic.config.yaml b/bootstrap/k3s/config/scorecard/patches/basic.config.yaml deleted file mode 100644 index 4a6c8167d..000000000 --- a/bootstrap/k3s/config/scorecard/patches/basic.config.yaml +++ /dev/null @@ -1,10 +0,0 @@ -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - basic-check-spec - image: quay.io/operator-framework/scorecard-test:v1.22.2 - labels: - suite: basic - test: basic-check-spec-test diff --git a/bootstrap/k3s/config/scorecard/patches/olm.config.yaml b/bootstrap/k3s/config/scorecard/patches/olm.config.yaml deleted file mode 100644 index c342410a9..000000000 --- a/bootstrap/k3s/config/scorecard/patches/olm.config.yaml +++ /dev/null @@ -1,50 +0,0 @@ -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-bundle-validation - image: quay.io/operator-framework/scorecard-test:v1.22.2 - labels: - suite: olm - test: olm-bundle-validation-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-crds-have-validation - image: quay.io/operator-framework/scorecard-test:v1.22.2 - labels: - suite: olm - test: olm-crds-have-validation-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-crds-have-resources - image: quay.io/operator-framework/scorecard-test:v1.22.2 - labels: - suite: olm - test: olm-crds-have-resources-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-spec-descriptors - image: quay.io/operator-framework/scorecard-test:v1.22.2 - labels: - suite: olm - test: olm-spec-descriptors-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-status-descriptors - image: quay.io/operator-framework/scorecard-test:v1.22.2 - labels: - suite: olm - test: olm-status-descriptors-test diff --git a/bootstrap/k3s/config/webhook/kustomization.yaml b/bootstrap/k3s/config/webhook/kustomization.yaml deleted file mode 100644 index 9cf26134e..000000000 --- a/bootstrap/k3s/config/webhook/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -resources: -- manifests.yaml -- service.yaml - -configurations: -- kustomizeconfig.yaml diff --git a/bootstrap/k3s/config/webhook/kustomizeconfig.yaml b/bootstrap/k3s/config/webhook/kustomizeconfig.yaml deleted file mode 100644 index 25e21e3c9..000000000 --- a/bootstrap/k3s/config/webhook/kustomizeconfig.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# the following config is for teaching kustomize where to look at when substituting vars. -# It requires kustomize v2.1.0 or newer to work properly. -nameReference: -- kind: Service - version: v1 - fieldSpecs: - - kind: MutatingWebhookConfiguration - group: admissionregistration.k8s.io - path: webhooks/clientConfig/service/name - - kind: ValidatingWebhookConfiguration - group: admissionregistration.k8s.io - path: webhooks/clientConfig/service/name - -namespace: -- kind: MutatingWebhookConfiguration - group: admissionregistration.k8s.io - path: webhooks/clientConfig/service/namespace - create: true -- kind: ValidatingWebhookConfiguration - group: admissionregistration.k8s.io - path: webhooks/clientConfig/service/namespace - create: true - -varReference: -- path: metadata/annotations diff --git a/bootstrap/k3s/config/webhook/manifests.yaml b/bootstrap/k3s/config/webhook/manifests.yaml deleted file mode 100644 index 8caf87493..000000000 --- a/bootstrap/k3s/config/webhook/manifests.yaml +++ /dev/null @@ -1,100 +0,0 @@ ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - creationTimestamp: null - name: mutating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /mutate-bootstrap-cluster-x-k8s-io-v1beta1-k3sconfig - failurePolicy: Fail - name: default.k3sconfig.bootstrap.cluster.x-k8s.io - rules: - - apiGroups: - - bootstrap.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - k3sconfigs - sideEffects: None -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /mutate-bootstrap-cluster-x-k8s-io-v1beta1-k3sconfigtemplate - failurePolicy: Fail - name: default.k3sconfigtemplate.bootstrap.cluster.x-k8s.io - rules: - - apiGroups: - - bootstrap.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - k3sconfigtemplates - sideEffects: None ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - creationTimestamp: null - name: validating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /validate-bootstrap-cluster-x-k8s-io-v1beta1-k3sconfig - failurePolicy: Fail - matchPolicy: Equivalent - name: validation.k3sconfig.bootstrap.cluster.x-k8s.io - rules: - - apiGroups: - - bootstrap.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - k3sconfigs - sideEffects: None -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /validate-bootstrap-cluster-x-k8s-io-v1beta1-k3sconfigtemplate - failurePolicy: Fail - matchPolicy: Equivalent - name: validation.k3sconfigtemplate.bootstrap.cluster.x-k8s.io - rules: - - apiGroups: - - bootstrap.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - k3sconfigtemplates - sideEffects: None diff --git a/bootstrap/k3s/config/webhook/service.yaml b/bootstrap/k3s/config/webhook/service.yaml deleted file mode 100644 index 3f638bd9c..000000000 --- a/bootstrap/k3s/config/webhook/service.yaml +++ /dev/null @@ -1,13 +0,0 @@ - -apiVersion: v1 -kind: Service -metadata: - name: webhook-service - namespace: system -spec: - ports: - - port: 443 - protocol: TCP - targetPort: 9443 - selector: - control-plane: controller-manager diff --git a/bootstrap/k3s/controllers/doc.go b/bootstrap/k3s/controllers/doc.go deleted file mode 100644 index 98b7479a2..000000000 --- a/bootstrap/k3s/controllers/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package controllers contains k3s config controllers. -package controllers diff --git a/bootstrap/k3s/controllers/k3sconfig_controller.go b/bootstrap/k3s/controllers/k3sconfig_controller.go deleted file mode 100644 index 5c953ca9b..000000000 --- a/bootstrap/k3s/controllers/k3sconfig_controller.go +++ /dev/null @@ -1,794 +0,0 @@ -/* -Copyright 2022. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - "context" - "fmt" - "time" - - "github.com/go-logr/logr" - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - kerrors "k8s.io/apimachinery/pkg/util/errors" - bootstraputil "k8s.io/cluster-bootstrap/token/util" - "k8s.io/klog/v2" - "k8s.io/utils/pointer" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" - bsutil "sigs.k8s.io/cluster-api/bootstrap/util" - expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" - "sigs.k8s.io/cluster-api/feature" - "sigs.k8s.io/cluster-api/util" - "sigs.k8s.io/cluster-api/util/annotations" - "sigs.k8s.io/cluster-api/util/conditions" - "sigs.k8s.io/cluster-api/util/patch" - "sigs.k8s.io/cluster-api/util/predicates" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/source" - - infrabootstrapv1 "github.com/kubesphere/kubekey/v3/bootstrap/k3s/api/v1beta1" - "github.com/kubesphere/kubekey/v3/bootstrap/k3s/pkg/cloudinit" - "github.com/kubesphere/kubekey/v3/bootstrap/k3s/pkg/locking" - k3stypes "github.com/kubesphere/kubekey/v3/bootstrap/k3s/pkg/types" - kklog "github.com/kubesphere/kubekey/v3/util/log" - "github.com/kubesphere/kubekey/v3/util/secret" -) - -// InitLocker is a lock that is used around kubeadm init. -type InitLocker interface { - Lock(ctx context.Context, cluster *clusterv1.Cluster, machine *clusterv1.Machine) bool - Unlock(ctx context.Context, cluster *clusterv1.Cluster) bool -} - -// +kubebuilder:rbac:groups=bootstrap.cluster.x-k8s.io,resources=k3sconfigs;k3sconfigs/status;k3sconfigs/finalizers,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;clusters/status;machinesets;machines;machines/status;machinepools;machinepools/status,verbs=get;list;watch -// +kubebuilder:rbac:groups="",resources=secrets;events;configmaps,verbs=get;list;watch;create;update;patch;delete - -// K3sConfigReconciler reconciles a K3sConfig object -type K3sConfigReconciler struct { - client.Client - K3sInitLock InitLocker - - // WatchFilterValue is the label value used to filter events prior to reconciliation. - WatchFilterValue string -} - -// Scope is a scoped struct used during reconciliation. -type Scope struct { - logr.Logger - Config *infrabootstrapv1.K3sConfig - ConfigOwner *bsutil.ConfigOwner - Cluster *clusterv1.Cluster -} - -// SetupWithManager sets up the controller with the Manager. -func (r *K3sConfigReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { - if r.K3sInitLock == nil { - r.K3sInitLock = locking.NewControlPlaneInitMutex(mgr.GetClient()) - } - - b := ctrl.NewControllerManagedBy(mgr). - For(&infrabootstrapv1.K3sConfig{}). - WithOptions(options). - Watches( - &source.Kind{Type: &clusterv1.Machine{}}, - handler.EnqueueRequestsFromMapFunc(r.MachineToBootstrapMapFunc), - ).WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(ctrl.LoggerFrom(ctx), r.WatchFilterValue)) - - if feature.Gates.Enabled(feature.MachinePool) { - b = b.Watches( - &source.Kind{Type: &expv1.MachinePool{}}, - handler.EnqueueRequestsFromMapFunc(r.MachinePoolToBootstrapMapFunc), - ).WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(ctrl.LoggerFrom(ctx), r.WatchFilterValue)) - } - - c, err := b.Build(r) - if err != nil { - return errors.Wrap(err, "failed setting up with a controller manager") - } - - err = c.Watch( - &source.Kind{Type: &clusterv1.Cluster{}}, - handler.EnqueueRequestsFromMapFunc(r.ClusterToK3sConfigs), - predicates.All(ctrl.LoggerFrom(ctx), - predicates.ClusterUnpausedAndInfrastructureReady(ctrl.LoggerFrom(ctx)), - predicates.ResourceHasFilterLabel(ctrl.LoggerFrom(ctx), r.WatchFilterValue), - ), - ) - if err != nil { - return errors.Wrap(err, "failed adding Watch for Clusters to controller manager") - } - - return nil -} - -// Reconcile handles K3sConfig events. -func (r *K3sConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, retErr error) { - log := ctrl.LoggerFrom(ctx) - - // Lookup the kubeadm config - config := &infrabootstrapv1.K3sConfig{} - if err := r.Client.Get(ctx, req.NamespacedName, config); err != nil { - if apierrors.IsNotFound(err) { - return ctrl.Result{}, nil - } - log.Error(err, "Failed to get config") - return ctrl.Result{}, err - } - - // AddOwners adds the owners of K3sConfig as k/v pairs to the logger. - // Specifically, it will add K3sControlPlane, MachineSet and MachineDeployment. - ctx, log, err := kklog.AddOwners(ctx, r.Client, config) - if err != nil { - return ctrl.Result{}, err - } - - // Look up the owner of this k3s config if there is one - configOwner, err := bsutil.GetConfigOwner(ctx, r.Client, config) - if apierrors.IsNotFound(err) { - // Could not find the owner yet, this is not an error and will rereconcile when the owner gets set. - return ctrl.Result{}, nil - } - if err != nil { - log.Error(err, "Failed to get owner") - return ctrl.Result{}, err - } - if configOwner == nil { - return ctrl.Result{}, nil - } - log = log.WithValues(configOwner.GetKind(), klog.KRef(configOwner.GetNamespace(), configOwner.GetName()), "resourceVersion", configOwner.GetResourceVersion()) - - log = log.WithValues("Cluster", klog.KRef(configOwner.GetNamespace(), configOwner.ClusterName())) - ctx = ctrl.LoggerInto(ctx, log) - - // Lookup the cluster the config owner is associated with - cluster, err := util.GetClusterByName(ctx, r.Client, configOwner.GetNamespace(), configOwner.ClusterName()) - if err != nil { - if errors.Cause(err) == util.ErrNoCluster { - log.Info(fmt.Sprintf("%s does not belong to a cluster yet, waiting until it's part of a cluster", configOwner.GetKind())) - return ctrl.Result{}, nil - } - - if apierrors.IsNotFound(err) { - log.Info("Cluster does not exist yet, waiting until it is created") - return ctrl.Result{}, nil - } - log.Error(err, "Could not get cluster with metadata") - return ctrl.Result{}, err - } - - if annotations.IsPaused(cluster, config) { - log.Info("Reconciliation is paused for this object") - return ctrl.Result{}, nil - } - - scope := &Scope{ - Logger: log, - Config: config, - ConfigOwner: configOwner, - Cluster: cluster, - } - - // Initialize the patch helper. - patchHelper, err := patch.NewHelper(config, r.Client) - if err != nil { - return ctrl.Result{}, err - } - - // Attempt to Patch the K3sConfig object and status after each reconciliation if no error occurs. - defer func() { - // always update the readyCondition; the summary is represented using the "1 of x completed" notation. - conditions.SetSummary(config, - conditions.WithConditions( - bootstrapv1.DataSecretAvailableCondition, - bootstrapv1.CertificatesAvailableCondition, - ), - ) - // Patch ObservedGeneration only if the reconciliation completed successfully - var patchOpts []patch.Option - if retErr == nil { - patchOpts = append(patchOpts, patch.WithStatusObservedGeneration{}) - } - if err := patchHelper.Patch(ctx, config, patchOpts...); err != nil { - log.Error(retErr, "Failed to patch config") - if retErr == nil { - retErr = err - } - } - }() - - switch { - // Wait for the infrastructure to be ready. - case !cluster.Status.InfrastructureReady: - log.Info("Cluster infrastructure is not ready, waiting") - conditions.MarkFalse(config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.WaitingForClusterInfrastructureReason, clusterv1.ConditionSeverityInfo, "") - return ctrl.Result{}, nil - // Reconcile status for machines that already have a secret reference, but our status isn't up-to-date. - // This case solves the pivoting scenario (or a backup restore) which doesn't preserve the status subresource on objects. - case configOwner.DataSecretName() != nil && (!config.Status.Ready || config.Status.DataSecretName == nil): - config.Status.Ready = true - config.Status.DataSecretName = configOwner.DataSecretName() - conditions.MarkTrue(config, bootstrapv1.DataSecretAvailableCondition) - return ctrl.Result{}, nil - // Status is ready means a config has been generated. - case config.Status.Ready: - return ctrl.Result{}, nil - } - - // Note: can't use IsFalse here because we need to handle the absence of the condition as well as false. - if !conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) { - return r.handleClusterNotInitialized(ctx, scope) - } - - // Every other case it's a join scenario - // Nb. in this case ClusterConfiguration and InitConfiguration should not be defined by users, but in case of misconfigurations, CABPK3s simply ignore them - - // Unlock any locks that might have been set during init process - r.K3sInitLock.Unlock(ctx, cluster) - - // if the .spec.cluster is missing, create a default one - if config.Spec.Cluster == nil { - log.Info("Creating default .spec.cluster") - config.Spec.Cluster = &infrabootstrapv1.Cluster{} - } - - // it's a control plane join - if configOwner.IsControlPlaneMachine() { - return r.joinControlplane(ctx, scope) - } - - // It's a worker join - return r.joinWorker(ctx, scope) -} - -func (r *K3sConfigReconciler) handleClusterNotInitialized(ctx context.Context, scope *Scope) (_ ctrl.Result, retErr error) { - // initialize the DataSecretAvailableCondition if missing. - // this is required in order to avoid the condition's LastTransitionTime to flicker in case of errors surfacing - // using the DataSecretGeneratedFailedReason - if conditions.GetReason(scope.Config, bootstrapv1.DataSecretAvailableCondition) != bootstrapv1.DataSecretGenerationFailedReason { - conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, clusterv1.WaitingForControlPlaneAvailableReason, clusterv1.ConditionSeverityInfo, "") - } - - // if it's NOT a control plane machine, requeue - if !scope.ConfigOwner.IsControlPlaneMachine() { - return ctrl.Result{RequeueAfter: 30 * time.Second}, nil - } - - // if the machine has not ClusterConfiguration and InitConfiguration, requeue - if scope.Config.Spec.ServerConfiguration == nil && scope.Config.Spec.Cluster == nil { - scope.Info("Control plane is not ready, requeing joining control planes until ready.") - return ctrl.Result{RequeueAfter: 30 * time.Second}, nil - } - - machine := &clusterv1.Machine{} - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(scope.ConfigOwner.Object, machine); err != nil { - return ctrl.Result{}, errors.Wrapf(err, "cannot convert %s to Machine", scope.ConfigOwner.GetKind()) - } - - // acquire the init lock so that only the first machine configured - // as control plane get processed here - // if not the first, requeue - if !r.K3sInitLock.Lock(ctx, scope.Cluster, machine) { - scope.Info("A control plane is already being initialized, requeing until control plane is ready") - return ctrl.Result{RequeueAfter: 30 * time.Second}, nil - } - - defer func() { - if retErr != nil { - if !r.K3sInitLock.Unlock(ctx, scope.Cluster) { - retErr = kerrors.NewAggregate([]error{retErr, errors.New("failed to unlock the kubeadm init lock")}) - } - } - }() - - scope.Info("Creating BootstrapData for the first control plane") - - if scope.Config.Spec.ServerConfiguration == nil { - scope.Config.Spec.ServerConfiguration = &infrabootstrapv1.ServerConfiguration{} - } - - // injects into config.ClusterConfiguration values from top level object - r.reconcileTopLevelObjectSettings(ctx, scope.Cluster, machine, scope.Config) - - certificates := secret.NewCertificatesForInitialControlPlane() - err := certificates.LookupOrGenerate( - ctx, - r.Client, - util.ObjectKey(scope.Cluster), - *metav1.NewControllerRef(scope.Config, infrabootstrapv1.GroupVersion.WithKind("K3sConfig")), - ) - if err != nil { - conditions.MarkFalse(scope.Config, bootstrapv1.CertificatesAvailableCondition, bootstrapv1.CertificatesGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error()) - return ctrl.Result{}, err - } - - conditions.MarkTrue(scope.Config, bootstrapv1.CertificatesAvailableCondition) - - t, err := r.generateAndStoreToken(ctx, scope) - if err != nil { - return ctrl.Result{}, err - } - - initData, err := k3stypes.MarshalInitServerConfiguration(&scope.Config.Spec, t) - if err != nil { - scope.Error(err, "Failed to marshal server configuration") - return ctrl.Result{}, err - } - - files, err := r.resolveFiles(ctx, scope.Config) - if err != nil { - conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.DataSecretGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error()) - return ctrl.Result{}, err - } - - initConfigFile := bootstrapv1.File{ - Path: k3stypes.DefaultK3sConfigLocation, - Content: initData, - Owner: "root:root", - Permissions: "0640", - } - - controlPlaneInput := &cloudinit.ControlPlaneInput{ - BaseUserData: cloudinit.BaseUserData{ - AdditionalFiles: files, - PreK3sCommands: scope.Config.Spec.PreK3sCommands, - PostK3sCommands: scope.Config.Spec.PostK3sCommands, - ConfigFile: initConfigFile, - }, - Certificates: certificates, - } - - bootstrapInitData, err := cloudinit.NewInitControlPlane(controlPlaneInput) - if err != nil { - scope.Error(err, "Failed to generate user data for bootstrap control plane") - return ctrl.Result{}, err - } - - if err := r.storeBootstrapData(ctx, scope, bootstrapInitData); err != nil { - scope.Error(err, "Failed to store bootstrap data") - return ctrl.Result{}, err - } - - return ctrl.Result{}, nil -} - -func (r *K3sConfigReconciler) joinWorker(ctx context.Context, scope *Scope) (ctrl.Result, error) { - scope.Info("Creating BootstrapData for the worker node") - - machine := &clusterv1.Machine{} - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(scope.ConfigOwner.Object, machine); err != nil { - return ctrl.Result{}, errors.Wrapf(err, "cannot convert %s to Machine", scope.ConfigOwner.GetKind()) - } - - // injects into config.Spec values from top level object - r.reconcileWorkerTopLevelObjectSettings(ctx, scope.Cluster, machine, scope.Config) - - // Ensure that agentConfiguration is properly set for joining node on the current cluster. - if res, err := r.reconcileDiscovery(ctx, scope.Cluster, scope.Config); err != nil { - return ctrl.Result{}, err - } else if !res.IsZero() { - return res, nil - } - - if scope.Config.Spec.AgentConfiguration == nil { - scope.Config.Spec.AgentConfiguration = &infrabootstrapv1.AgentConfiguration{} - } - - joinWorkerData, err := k3stypes.MarshalJoinAgentConfiguration(&scope.Config.Spec) - if err != nil { - scope.Error(err, "Failed to marshal join configuration") - return ctrl.Result{}, err - } - - files, err := r.resolveFiles(ctx, scope.Config) - if err != nil { - conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.DataSecretGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error()) - return ctrl.Result{}, err - } - - joinConfigFile := bootstrapv1.File{ - Path: k3stypes.DefaultK3sConfigLocation, - Content: joinWorkerData, - Owner: "root:root", - Permissions: "0640", - } - - workerJoinInput := &cloudinit.NodeInput{ - BaseUserData: cloudinit.BaseUserData{ - AdditionalFiles: files, - PreK3sCommands: scope.Config.Spec.PreK3sCommands, - PostK3sCommands: scope.Config.Spec.PostK3sCommands, - ConfigFile: joinConfigFile, - }, - } - - cloudInitData, err := cloudinit.NewNode(workerJoinInput) - if err != nil { - scope.Error(err, "Failed to generate user data for bootstrap control plane") - return ctrl.Result{}, err - } - - if err := r.storeBootstrapData(ctx, scope, cloudInitData); err != nil { - scope.Error(err, "Failed to store bootstrap data") - return ctrl.Result{}, err - } - return ctrl.Result{}, nil -} - -func (r *K3sConfigReconciler) joinControlplane(ctx context.Context, scope *Scope) (ctrl.Result, error) { - scope.Info("Creating BootstrapData for the joining control plane") - - if !scope.ConfigOwner.IsControlPlaneMachine() { - return ctrl.Result{}, fmt.Errorf("%s is not a valid control plane kind, only Machine is supported", scope.ConfigOwner.GetKind()) - } - - if scope.Config.Spec.ServerConfiguration == nil { - scope.Config.Spec.ServerConfiguration = &infrabootstrapv1.ServerConfiguration{} - } - - machine := &clusterv1.Machine{} - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(scope.ConfigOwner.Object, machine); err != nil { - return ctrl.Result{}, errors.Wrapf(err, "cannot convert %s to Machine", scope.ConfigOwner.GetKind()) - } - - // injects into config.ClusterConfiguration values from top level object - r.reconcileTopLevelObjectSettings(ctx, scope.Cluster, machine, scope.Config) - - // Ensure that joinConfiguration.Discovery is properly set for joining node on the current cluster. - if res, err := r.reconcileDiscovery(ctx, scope.Cluster, scope.Config); err != nil { - return ctrl.Result{}, err - } else if !res.IsZero() { - return res, nil - } - - joinData, err := k3stypes.MarshalJoinServerConfiguration(&scope.Config.Spec) - if err != nil { - scope.Error(err, "Failed to marshal join configuration") - return ctrl.Result{}, err - } - - files, err := r.resolveFiles(ctx, scope.Config) - if err != nil { - conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.DataSecretGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error()) - return ctrl.Result{}, err - } - - joinConfigFile := bootstrapv1.File{ - Path: k3stypes.DefaultK3sConfigLocation, - Content: joinData, - Owner: "root:root", - Permissions: "0640", - } - - controlPlaneJoinInput := &cloudinit.ControlPlaneInput{ - BaseUserData: cloudinit.BaseUserData{ - AdditionalFiles: files, - PreK3sCommands: scope.Config.Spec.PreK3sCommands, - PostK3sCommands: scope.Config.Spec.PostK3sCommands, - ConfigFile: joinConfigFile, - }, - } - - cloudInitData, err := cloudinit.NewJoinControlPlane(controlPlaneJoinInput) - if err != nil { - scope.Error(err, "Failed to generate user data for bootstrap control plane") - return ctrl.Result{}, err - } - - if err := r.storeBootstrapData(ctx, scope, cloudInitData); err != nil { - scope.Error(err, "Failed to store bootstrap data") - return ctrl.Result{}, err - } - return ctrl.Result{}, nil -} - -func (r *K3sConfigReconciler) generateAndStoreToken(ctx context.Context, scope *Scope) (string, error) { - t, err := bootstraputil.GenerateBootstrapToken() - if err != nil { - return "", errors.Wrap(err, "unable to generate bootstrap token") - } - - s := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-token", scope.Cluster.Name), - Namespace: scope.Config.Namespace, - Labels: map[string]string{ - clusterv1.ClusterLabelName: scope.Cluster.Name, - }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: infrabootstrapv1.GroupVersion.String(), - Kind: "K3sConfig", - Name: scope.Config.Name, - UID: scope.Config.UID, - Controller: pointer.Bool(true), - }, - }, - }, - Data: map[string][]byte{ - "value": []byte(t), - }, - Type: clusterv1.ClusterSecretType, - } - - // as secret creation and scope.Config status patch are not atomic operations - // it is possible that secret creation happens but the config.Status patches are not applied - if err := r.Client.Create(ctx, s); err != nil { - if !apierrors.IsAlreadyExists(err) { - return "", errors.Wrapf(err, "failed to create token for K3sConfig %s/%s", scope.Config.Namespace, scope.Config.Name) - } - if err := r.Client.Update(ctx, s); err != nil { - return "", errors.Wrapf(err, "failed to update bootstrap token secret for K3sConfig %s/%s", scope.Config.Namespace, scope.Config.Name) - } - } - - return t, nil -} - -// resolveFiles maps .Spec.Files into cloudinit.Files, resolving any object references -// along the way. -func (r *K3sConfigReconciler) resolveFiles(ctx context.Context, cfg *infrabootstrapv1.K3sConfig) ([]bootstrapv1.File, error) { - collected := make([]bootstrapv1.File, 0, len(cfg.Spec.Files)) - - for i := range cfg.Spec.Files { - in := cfg.Spec.Files[i] - if in.ContentFrom != nil { - data, err := r.resolveSecretFileContent(ctx, cfg.Namespace, in) - if err != nil { - return nil, errors.Wrapf(err, "failed to resolve file source") - } - in.ContentFrom = nil - in.Content = string(data) - } - collected = append(collected, in) - } - - return collected, nil -} - -// resolveSecretFileContent returns file content fetched from a referenced secret object. -func (r *K3sConfigReconciler) resolveSecretFileContent(ctx context.Context, ns string, source bootstrapv1.File) ([]byte, error) { - s := &corev1.Secret{} - key := types.NamespacedName{Namespace: ns, Name: source.ContentFrom.Secret.Name} - if err := r.Client.Get(ctx, key, s); err != nil { - if apierrors.IsNotFound(err) { - return nil, errors.Wrapf(err, "secret not found: %s", key) - } - return nil, errors.Wrapf(err, "failed to retrieve Secret %q", key) - } - data, ok := s.Data[source.ContentFrom.Secret.Key] - if !ok { - return nil, errors.Errorf("secret references non-existent secret key: %q", source.ContentFrom.Secret.Key) - } - return data, nil -} - -// storeBootstrapData creates a new secret with the data passed in as input, -// sets the reference in the configuration status and ready to true. -func (r *K3sConfigReconciler) storeBootstrapData(ctx context.Context, scope *Scope, data []byte) error { - log := ctrl.LoggerFrom(ctx) - - s := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: scope.Config.Name, - Namespace: scope.Config.Namespace, - Labels: map[string]string{ - clusterv1.ClusterLabelName: scope.Cluster.Name, - }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: infrabootstrapv1.GroupVersion.String(), - Kind: "K3sConfig", - Name: scope.Config.Name, - UID: scope.Config.UID, - Controller: pointer.Bool(true), - }, - }, - }, - Data: map[string][]byte{ - "value": data, - }, - Type: clusterv1.ClusterSecretType, - } - - // as secret creation and scope.Config status patch are not atomic operations - // it is possible that secret creation happens but the config.Status patches are not applied - if err := r.Client.Create(ctx, s); err != nil { - if !apierrors.IsAlreadyExists(err) { - return errors.Wrapf(err, "failed to create bootstrap data secret for K3sConfig %s/%s", scope.Config.Namespace, scope.Config.Name) - } - log.Info("bootstrap data secret for K3sConfig already exists, updating", "Secret", klog.KObj(s)) - if err := r.Client.Update(ctx, s); err != nil { - return errors.Wrapf(err, "failed to update bootstrap data secret for K3sConfig %s/%s", scope.Config.Namespace, scope.Config.Name) - } - } - scope.Config.Status.DataSecretName = pointer.String(s.Name) - scope.Config.Status.Ready = true - conditions.MarkTrue(scope.Config, bootstrapv1.DataSecretAvailableCondition) - return nil -} - -func (r *K3sConfigReconciler) reconcileDiscovery(ctx context.Context, cluster *clusterv1.Cluster, config *infrabootstrapv1.K3sConfig) (ctrl.Result, error) { - log := ctrl.LoggerFrom(ctx) - - // if config already contains a file discovery configuration, respect it without further validations - if config.Spec.Cluster.TokenFile != "" { - return ctrl.Result{}, nil - } - - // if BootstrapToken already contains an APIServerEndpoint, respect it; otherwise inject the APIServerEndpoint endpoint defined in cluster status - apiServerEndpoint := config.Spec.Cluster.Server - if apiServerEndpoint == "" { - if !cluster.Spec.ControlPlaneEndpoint.IsValid() { - log.V(1).Info("Waiting for Cluster Controller to set Cluster.Server") - return ctrl.Result{RequeueAfter: 10 * time.Second}, nil - } - - apiServerEndpoint = cluster.Spec.ControlPlaneEndpoint.String() - config.Spec.Cluster.Server = fmt.Sprintf("https://%s", apiServerEndpoint) - log.V(3).Info("Altering Cluster.Server", "Server", apiServerEndpoint) - } - - // if BootstrapToken already contains a token, respect it; otherwise create a new bootstrap token for the node to join - if config.Spec.Cluster.Token == "" { - s := &corev1.Secret{} - obj := client.ObjectKey{ - Namespace: config.Namespace, - Name: fmt.Sprintf("%s-token", cluster.Name), - } - - if err := r.Client.Get(ctx, obj, s); err != nil { - return ctrl.Result{}, errors.Wrapf(err, "failed to get token for K3sConfig %s/%s", config.Namespace, config.Name) - } - - config.Spec.Cluster.Token = string(s.Data["value"]) - log.V(3).Info("Altering Cluster.Token") - } - - return ctrl.Result{}, nil -} - -// MachineToBootstrapMapFunc is a handler.ToRequestsFunc to be used to enqueue -// request for reconciliation of K3sConfig. -func (r *K3sConfigReconciler) MachineToBootstrapMapFunc(o client.Object) []ctrl.Request { - m, ok := o.(*clusterv1.Machine) - if !ok { - panic(fmt.Sprintf("Expected a Machine but got a %T", o)) - } - - var result []ctrl.Request - if m.Spec.Bootstrap.ConfigRef != nil && m.Spec.Bootstrap.ConfigRef.GroupVersionKind() == infrabootstrapv1.GroupVersion.WithKind("K3sConfig") { - name := client.ObjectKey{Namespace: m.Namespace, Name: m.Spec.Bootstrap.ConfigRef.Name} - result = append(result, ctrl.Request{NamespacedName: name}) - } - return result -} - -// MachinePoolToBootstrapMapFunc is a handler.ToRequestsFunc to be used to enqueue -// request for reconciliation of K3sConfig. -func (r *K3sConfigReconciler) MachinePoolToBootstrapMapFunc(o client.Object) []ctrl.Request { - m, ok := o.(*expv1.MachinePool) - if !ok { - panic(fmt.Sprintf("Expected a MachinePool but got a %T", o)) - } - - var result []ctrl.Request - configRef := m.Spec.Template.Spec.Bootstrap.ConfigRef - if configRef != nil && configRef.GroupVersionKind().GroupKind() == infrabootstrapv1.GroupVersion.WithKind("K3sConfig").GroupKind() { - name := client.ObjectKey{Namespace: m.Namespace, Name: configRef.Name} - result = append(result, ctrl.Request{NamespacedName: name}) - } - return result -} - -// ClusterToK3sConfigs is a handler.ToRequestsFunc to be used to enqueue -// requests for reconciliation of K3sConfig. -func (r *K3sConfigReconciler) ClusterToK3sConfigs(o client.Object) []ctrl.Request { - var result []ctrl.Request - - c, ok := o.(*clusterv1.Cluster) - if !ok { - panic(fmt.Sprintf("Expected a Cluster but got a %T", o)) - } - - selectors := []client.ListOption{ - client.InNamespace(c.Namespace), - client.MatchingLabels{ - clusterv1.ClusterLabelName: c.Name, - }, - } - - machineList := &clusterv1.MachineList{} - if err := r.Client.List(context.TODO(), machineList, selectors...); err != nil { - return nil - } - - for _, m := range machineList.Items { - if m.Spec.Bootstrap.ConfigRef != nil && - m.Spec.Bootstrap.ConfigRef.GroupVersionKind().GroupKind() == infrabootstrapv1.GroupVersion.WithKind("K3sConfig").GroupKind() { - name := client.ObjectKey{Namespace: m.Namespace, Name: m.Spec.Bootstrap.ConfigRef.Name} - result = append(result, ctrl.Request{NamespacedName: name}) - } - } - - if feature.Gates.Enabled(feature.MachinePool) { - machinePoolList := &expv1.MachinePoolList{} - if err := r.Client.List(context.TODO(), machinePoolList, selectors...); err != nil { - return nil - } - - for _, mp := range machinePoolList.Items { - if mp.Spec.Template.Spec.Bootstrap.ConfigRef != nil && - mp.Spec.Template.Spec.Bootstrap.ConfigRef.GroupVersionKind().GroupKind() == infrabootstrapv1.GroupVersion.WithKind("K3sConfig").GroupKind() { - name := client.ObjectKey{Namespace: mp.Namespace, Name: mp.Spec.Template.Spec.Bootstrap.ConfigRef.Name} - result = append(result, ctrl.Request{NamespacedName: name}) - } - } - } - - return result -} - -// reconcileTopLevelObjectSettings injects into config.ClusterConfiguration values from top level objects like cluster and machine. -// The implementation func respect user provided config values, but in case some of them are missing, values from top level objects are used. -func (r *K3sConfigReconciler) reconcileTopLevelObjectSettings(ctx context.Context, cluster *clusterv1.Cluster, machine *clusterv1.Machine, config *infrabootstrapv1.K3sConfig) { - log := ctrl.LoggerFrom(ctx) - - // If there are no Network settings defined in ClusterConfiguration, use ClusterNetwork settings, if defined - if cluster.Spec.ClusterNetwork != nil { - if config.Spec.ServerConfiguration.Networking.ClusterDomain == "" && cluster.Spec.ClusterNetwork.ServiceDomain != "" { - config.Spec.ServerConfiguration.Networking.ClusterDomain = cluster.Spec.ClusterNetwork.ServiceDomain - log.V(3).Info("Altering ServerConfiguration.Networking.ClusterDomain", "ClusterDomain", config.Spec.ServerConfiguration.Networking.ClusterDomain) - } - if config.Spec.ServerConfiguration.Networking.ServiceCIDR == "" && - cluster.Spec.ClusterNetwork.Services != nil && - len(cluster.Spec.ClusterNetwork.Services.CIDRBlocks) > 0 { - config.Spec.ServerConfiguration.Networking.ServiceCIDR = cluster.Spec.ClusterNetwork.Services.String() - log.V(3).Info("Altering ServerConfiguration.Networking.ServiceCIDR", "ServiceCIDR", config.Spec.ServerConfiguration.Networking.ServiceCIDR) - } - if config.Spec.ServerConfiguration.Networking.ClusterCIDR == "" && - cluster.Spec.ClusterNetwork.Pods != nil && - len(cluster.Spec.ClusterNetwork.Pods.CIDRBlocks) > 0 { - config.Spec.ServerConfiguration.Networking.ClusterCIDR = cluster.Spec.ClusterNetwork.Pods.String() - log.V(3).Info("Altering ServerConfiguration.Networking.ClusterCIDR", "ClusterCIDR", config.Spec.ServerConfiguration.Networking.ClusterCIDR) - } - } - - // If there are no Version settings defined, use Version from machine, if defined - if config.Spec.Version == "" && machine.Spec.Version != nil { - config.Spec.Version = *machine.Spec.Version - log.V(3).Info("Altering Spec.Version", "Version", config.Spec.Version) - } -} - -func (r *K3sConfigReconciler) reconcileWorkerTopLevelObjectSettings(ctx context.Context, _ *clusterv1.Cluster, machine *clusterv1.Machine, config *infrabootstrapv1.K3sConfig) { - log := ctrl.LoggerFrom(ctx) - - // If there are no Version settings defined, use Version from machine, if defined - if config.Spec.Version == "" && machine.Spec.Version != nil { - config.Spec.Version = *machine.Spec.Version - log.V(3).Info("Altering Spec.Version", "Version", config.Spec.Version) - } -} diff --git a/bootstrap/k3s/controllers/suite_test.go b/bootstrap/k3s/controllers/suite_test.go deleted file mode 100644 index 76505fda0..000000000 --- a/bootstrap/k3s/controllers/suite_test.go +++ /dev/null @@ -1,29 +0,0 @@ -/* -Copyright 2022. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -//var ( -// env *envtest.Environment -// ctx = ctrl.SetupSignalHandler() -//) -// -//func TestMain(m *testing.M) { -// os.Exit(envtest.Run(ctx, envtest.RunInput{ -// M: m, -// SetupEnv: func(e *envtest.Environment) { env = e }, -// })) -//} diff --git a/bootstrap/k3s/hack/boilerplate.go.txt b/bootstrap/k3s/hack/boilerplate.go.txt deleted file mode 100644 index 62802d182..000000000 --- a/bootstrap/k3s/hack/boilerplate.go.txt +++ /dev/null @@ -1,15 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ \ No newline at end of file diff --git a/bootstrap/k3s/main.go b/bootstrap/k3s/main.go deleted file mode 100644 index a994a1fed..000000000 --- a/bootstrap/k3s/main.go +++ /dev/null @@ -1,193 +0,0 @@ -/* -Copyright 2022. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package main -package main - -import ( - "flag" - "fmt" - "math/rand" - "os" - "time" - - "github.com/spf13/pflag" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - _ "k8s.io/client-go/plugin/pkg/client/auth" - "k8s.io/client-go/tools/leaderelection/resourcelock" - "k8s.io/klog/v2" - "k8s.io/klog/v2/klogr" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/controllers/remote" - expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" - "sigs.k8s.io/cluster-api/feature" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/healthz" - - infrabootstrapv1 "github.com/kubesphere/kubekey/v3/bootstrap/k3s/api/v1beta1" - "github.com/kubesphere/kubekey/v3/bootstrap/k3s/controllers" - infracontrolplanev1 "github.com/kubesphere/kubekey/v3/controlplane/k3s/api/v1beta1" - //+kubebuilder:scaffold:imports -) - -var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") -) - -func init() { - utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - utilruntime.Must(clusterv1.AddToScheme(scheme)) - utilruntime.Must(expv1.AddToScheme(scheme)) - utilruntime.Must(infrabootstrapv1.AddToScheme(scheme)) - utilruntime.Must(infracontrolplanev1.AddToScheme(scheme)) - //+kubebuilder:scaffold:scheme -} - -var ( - metricsAddr string - enableLeaderElection bool - leaderElectionLeaseDuration time.Duration - leaderElectionRenewDeadline time.Duration - leaderElectionRetryPeriod time.Duration - k3sConfigConcurrency int - healthAddr string - watchFilterValue string - watchNamespace string - syncPeriod time.Duration - webhookPort int - webhookCertDir string -) - -func main() { - klog.InitFlags(nil) - - rand.Seed(time.Now().UnixNano()) - initFlags(pflag.CommandLine) - pflag.CommandLine.AddGoFlagSet(flag.CommandLine) - pflag.Parse() - - ctrl.SetLogger(klogr.New()) - - ctx := ctrl.SetupSignalHandler() - - restConfig := ctrl.GetConfigOrDie() - restConfig.UserAgent = remote.DefaultClusterAPIUserAgent("cluster-api-k3s-bootstrap-manager") - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: metricsAddr, - LeaderElection: enableLeaderElection, - LeaderElectionID: "k3s-bootstrap-manager-leader-election-capkk", - LeaderElectionResourceLock: resourcelock.LeasesResourceLock, - LeaseDuration: &leaderElectionLeaseDuration, - RenewDeadline: &leaderElectionRenewDeadline, - RetryPeriod: &leaderElectionRetryPeriod, - SyncPeriod: &syncPeriod, - ClientDisableCacheFor: []client.Object{ - &corev1.ConfigMap{}, - &corev1.Secret{}, - }, - Namespace: watchNamespace, - Port: webhookPort, - HealthProbeBindAddress: healthAddr, - CertDir: webhookCertDir, - }) - if err != nil { - setupLog.Error(err, "unable to start manager") - os.Exit(1) - } - - if err := (&controllers.K3sConfigReconciler{ - Client: mgr.GetClient(), - WatchFilterValue: watchFilterValue, - }).SetupWithManager(ctx, mgr, concurrency(k3sConfigConcurrency)); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "K3sConfig") - os.Exit(1) - } - - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up health check") - os.Exit(1) - } - if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up ready check") - os.Exit(1) - } - - if err = (&infrabootstrapv1.K3sConfig{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "K3sConfig") - os.Exit(1) - } - if err = (&infrabootstrapv1.K3sConfigTemplate{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "K3sConfigTemplate") - os.Exit(1) - } - // +kubebuilder:scaffold:builder - setupLog.Info("starting manager") - if err := mgr.Start(ctx); err != nil { - setupLog.Error(err, "problem running manager") - os.Exit(1) - } -} - -func initFlags(fs *pflag.FlagSet) { - fs.StringVar(&metricsAddr, "metrics-bind-addr", "localhost:8080", - "The address the metric endpoint binds to.") - - fs.BoolVar(&enableLeaderElection, "leader-elect", false, - "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") - - fs.DurationVar(&leaderElectionLeaseDuration, "leader-elect-lease-duration", 15*time.Second, - "Interval at which non-leader candidates will wait to force acquire leadership (duration string)") - - fs.DurationVar(&leaderElectionRenewDeadline, "leader-elect-renew-deadline", 10*time.Second, - "Duration that the leading controller manager will retry refreshing leadership before giving up (duration string)") - - fs.DurationVar(&leaderElectionRetryPeriod, "leader-elect-retry-period", 2*time.Second, - "Duration the LeaderElector clients should wait between tries of actions (duration string)") - - fs.StringVar(&watchNamespace, "namespace", "", - "Namespace that the controller watches to reconcile cluster-api objects. If unspecified, the controller watches for cluster-api objects across all namespaces.") - - fs.StringVar(&healthAddr, "health-addr", ":9440", - "The address the health endpoint binds to.") - - fs.IntVar(&k3sConfigConcurrency, "k3sconfig-concurrency", 10, - "Number of kubeadm configs to process simultaneously") - - fs.DurationVar(&syncPeriod, "sync-period", 10*time.Minute, - "The minimum interval at which watched resources are reconciled (e.g. 15m)") - - fs.StringVar(&watchFilterValue, "watch-filter", "", - fmt.Sprintf("Label value that the controller watches to reconcile cluster-api objects. Label key is always %s. If unspecified, the controller watches for all cluster-api objects.", clusterv1.WatchLabel)) - - fs.IntVar(&webhookPort, "webhook-port", 9443, - "Webhook Server port") - - fs.StringVar(&webhookCertDir, "webhook-cert-dir", "/tmp/k8s-webhook-server/serving-certs/", - "Webhook cert dir, only used when webhook-port is specified.") - - feature.MutableGates.AddFlag(fs) -} - -func concurrency(c int) controller.Options { - return controller.Options{MaxConcurrentReconciles: c} -} diff --git a/bootstrap/k3s/pkg/cloudinit/cloudinit.go b/bootstrap/k3s/pkg/cloudinit/cloudinit.go deleted file mode 100644 index 098d94929..000000000 --- a/bootstrap/k3s/pkg/cloudinit/cloudinit.go +++ /dev/null @@ -1,76 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cloudinit - -import ( - "bytes" - _ "embed" - "text/template" - - "github.com/pkg/errors" - bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" -) - -const ( - // sentinelFileCommand writes a file to /run/cluster-api to signal successful Kubernetes bootstrapping in a way that - // works both for Linux and Windows OS. - sentinelFileCommand = "echo success > /run/cluster-api/bootstrap-success.complete" - cloudConfigHeader = `## template: jinja -#cloud-config -` -) - -// BaseUserData is shared across all the various types of files written to disk. -type BaseUserData struct { - Header string - PreK3sCommands []string - PostK3sCommands []string - AdditionalFiles []bootstrapv1.File - WriteFiles []bootstrapv1.File - ConfigFile bootstrapv1.File - SentinelFileCommand string -} - -func (input *BaseUserData) prepare() { - input.Header = cloudConfigHeader - input.WriteFiles = append(input.WriteFiles, input.AdditionalFiles...) - input.WriteFiles = append(input.WriteFiles, input.ConfigFile) - input.SentinelFileCommand = sentinelFileCommand -} - -func generate(kind string, tpl string, data interface{}) ([]byte, error) { - tm := template.New(kind).Funcs(defaultTemplateFuncMap) - if _, err := tm.Parse(filesTemplate); err != nil { - return nil, errors.Wrap(err, "failed to parse files template") - } - - if _, err := tm.Parse(commandsTemplate); err != nil { - return nil, errors.Wrap(err, "failed to parse commands template") - } - - t, err := tm.Parse(tpl) - if err != nil { - return nil, errors.Wrapf(err, "failed to parse %s template", kind) - } - - var out bytes.Buffer - if err := t.Execute(&out, data); err != nil { - return nil, errors.Wrapf(err, "failed to generate %s template", kind) - } - - return out.Bytes(), nil -} diff --git a/bootstrap/k3s/pkg/cloudinit/commands.go b/bootstrap/k3s/pkg/cloudinit/commands.go deleted file mode 100644 index 9fefb6d95..000000000 --- a/bootstrap/k3s/pkg/cloudinit/commands.go +++ /dev/null @@ -1,26 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cloudinit - -const ( - commandsTemplate = `{{- define "commands" -}} -{{ range . }} - - {{printf "%q" .}} -{{- end -}} -{{- end -}} -` -) diff --git a/bootstrap/k3s/pkg/cloudinit/controlplane_init.go b/bootstrap/k3s/pkg/cloudinit/controlplane_init.go deleted file mode 100644 index dc4d3b6c5..000000000 --- a/bootstrap/k3s/pkg/cloudinit/controlplane_init.go +++ /dev/null @@ -1,58 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cloudinit - -import ( - "github.com/kubesphere/kubekey/v3/util/secret" -) - -const ( - controlPlaneCloudInit = `{{.Header}} -{{template "files" .WriteFiles}} -- path: /run/cluster-api/placeholder - owner: root:root - permissions: '0640' - content: "This placeholder file is used to create the /run/cluster-api sub directory in a way that is compatible with both Linux and Windows (mkdir -p /run/cluster-api does not work with Windows)" -runcmd: -{{- template "commands" .PreK3sCommands }} - - "INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_EXEC='server' /usr/local/bin/k3s-install.sh" -{{- template "commands" .PostK3sCommands }} -` -) - -// ControlPlaneInput defines the context to generate a controlplane instance user data. -type ControlPlaneInput struct { - BaseUserData - secret.Certificates - - ServerConfiguration string -} - -// NewInitControlPlane returns the clouding string to be used on initializing a controlplane instance. -func NewInitControlPlane(input *ControlPlaneInput) ([]byte, error) { - input.Header = cloudConfigHeader - input.WriteFiles = input.Certificates.AsFiles() - input.WriteFiles = append(input.WriteFiles, input.AdditionalFiles...) - input.WriteFiles = append(input.WriteFiles, input.ConfigFile) - input.SentinelFileCommand = sentinelFileCommand - userData, err := generate("InitControlplane", controlPlaneCloudInit, input) - if err != nil { - return nil, err - } - - return userData, nil -} diff --git a/bootstrap/k3s/pkg/cloudinit/controlplane_join.go b/bootstrap/k3s/pkg/cloudinit/controlplane_join.go deleted file mode 100644 index 88ea61462..000000000 --- a/bootstrap/k3s/pkg/cloudinit/controlplane_join.go +++ /dev/null @@ -1,46 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cloudinit - -import ( - "github.com/pkg/errors" -) - -const ( - controlPlaneJoinCloudInit = `{{.Header}} -{{template "files" .WriteFiles}} -- path: /run/cluster-api/placeholder - owner: root:root - permissions: '0640' - content: "This placeholder file is used to create the /run/cluster-api sub directory in a way that is compatible with both Linux and Windows (mkdir -p /run/cluster-api does not work with Windows)" -runcmd: -{{- template "commands" .PreK3sCommands }} - - "INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_EXEC='server' /usr/local/bin/k3s-install.sh" -{{- template "commands" .PostK3sCommands }} -` -) - -// NewJoinControlPlane returns the cloudinit string to be used on joining a control plane instance. -func NewJoinControlPlane(input *ControlPlaneInput) ([]byte, error) { - input.prepare() - userData, err := generate("JoinControlplane", controlPlaneJoinCloudInit, input) - if err != nil { - return nil, errors.Wrapf(err, "failed to generate user data for machine joining control plane") - } - - return userData, err -} diff --git a/bootstrap/k3s/pkg/cloudinit/doc.go b/bootstrap/k3s/pkg/cloudinit/doc.go deleted file mode 100644 index 1a473c7de..000000000 --- a/bootstrap/k3s/pkg/cloudinit/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package cloudinit implements kubeadm cloudinit functionality. -package cloudinit diff --git a/bootstrap/k3s/pkg/cloudinit/files.go b/bootstrap/k3s/pkg/cloudinit/files.go deleted file mode 100644 index fd0c3f89b..000000000 --- a/bootstrap/k3s/pkg/cloudinit/files.go +++ /dev/null @@ -1,40 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cloudinit - -const ( - filesTemplate = `{{ define "files" -}} -write_files:{{ range . }} -- path: {{.Path}} - {{ if ne .Encoding "" -}} - encoding: "{{.Encoding}}" - {{ end -}} - {{ if ne .Owner "" -}} - owner: {{.Owner}} - {{ end -}} - {{ if ne .Permissions "" -}} - permissions: '{{.Permissions}}' - {{ end -}} - {{ if .Append -}} - append: true - {{ end -}} - content: | -{{.Content | Indent 6}} -{{- end -}} -{{- end -}} -` -) diff --git a/bootstrap/k3s/pkg/cloudinit/node.go b/bootstrap/k3s/pkg/cloudinit/node.go deleted file mode 100644 index 0335c9b94..000000000 --- a/bootstrap/k3s/pkg/cloudinit/node.go +++ /dev/null @@ -1,51 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cloudinit - -import ( - "github.com/pkg/errors" -) - -const ( - workerCloudInit = `{{.Header}} -{{template "files" .WriteFiles}} -- path: /run/cluster-api/placeholder - owner: root:root - permissions: '0640' - content: "This placeholder file is used to create the /run/cluster-api sub directory in a way that is compatible with both Linux and Windows (mkdir -p /run/cluster-api does not work with Windows)" -runcmd: -{{- template "commands" .PreK3sCommands }} - - "INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_EXEC='agent' /usr/local/bin/k3s-install.sh" -{{- template "commands" .PostK3sCommands }} -` -) - -// NodeInput defines the context to generate an agent node cloud-init. -type NodeInput struct { - BaseUserData -} - -// NewNode returns the cloud-init for joining a node instance. -func NewNode(input *NodeInput) ([]byte, error) { - input.prepare() - userData, err := generate("JoinWorker", workerCloudInit, input) - if err != nil { - return nil, errors.Wrapf(err, "failed to generate user data for machine joining worker node") - } - - return userData, err -} diff --git a/bootstrap/k3s/pkg/cloudinit/utils.go b/bootstrap/k3s/pkg/cloudinit/utils.go deleted file mode 100644 index 1bfd00377..000000000 --- a/bootstrap/k3s/pkg/cloudinit/utils.go +++ /dev/null @@ -1,34 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cloudinit - -import ( - "strings" - "text/template" -) - -var ( - defaultTemplateFuncMap = template.FuncMap{ - "Indent": templateYAMLIndent, - } -) - -func templateYAMLIndent(i int, input string) string { - split := strings.Split(input, "\n") - ident := "\n" + strings.Repeat(" ", i) - return strings.Repeat(" ", i) + strings.Join(split, ident) -} diff --git a/bootstrap/k3s/pkg/locking/control_plane_init_mutex.go b/bootstrap/k3s/pkg/locking/control_plane_init_mutex.go deleted file mode 100644 index fdc71822d..000000000 --- a/bootstrap/k3s/pkg/locking/control_plane_init_mutex.go +++ /dev/null @@ -1,190 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package locking implements locking functionality. -package locking - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/klog/v2" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const semaphoreInformationKey = "lock-information" - -// ControlPlaneInitMutex uses a ConfigMap to synchronize cluster initialization. -type ControlPlaneInitMutex struct { - client client.Client -} - -// NewControlPlaneInitMutex returns a lock that can be held by a control plane node before init. -func NewControlPlaneInitMutex(client client.Client) *ControlPlaneInitMutex { - return &ControlPlaneInitMutex{ - client: client, - } -} - -// Lock allows a control plane node to be the first and only node to run kubeadm init. -func (c *ControlPlaneInitMutex) Lock(ctx context.Context, cluster *clusterv1.Cluster, machine *clusterv1.Machine) bool { - sema := newSemaphore() - cmName := configMapName(cluster.Name) - log := ctrl.LoggerFrom(ctx, "ConfigMap", klog.KRef(cluster.Namespace, cmName)) - err := c.client.Get(ctx, client.ObjectKey{ - Namespace: cluster.Namespace, - Name: cmName, - }, sema.ConfigMap) - switch { - case apierrors.IsNotFound(err): - break - case err != nil: - log.Error(err, "Failed to acquire init lock") - return false - default: // Successfully found an existing config map. - info, err := sema.information() - if err != nil { - log.Error(err, "Failed to get information about the existing init lock") - return false - } - // The machine requesting the lock is the machine that created the lock, therefore the lock is acquired. - if info.MachineName == machine.Name { - return true - } - - // If the machine that created the lock can not be found unlock the mutex. - if err := c.client.Get(ctx, client.ObjectKey{ - Namespace: cluster.Namespace, - Name: info.MachineName, - }, &clusterv1.Machine{}); err != nil { - log.Error(err, "Failed to get machine holding init lock") - if apierrors.IsNotFound(err) { - c.Unlock(ctx, cluster) - } - } - log.Info(fmt.Sprintf("Waiting for Machine %s to initialize", info.MachineName)) - return false - } - - // Adds owner reference, namespace and name - sema.setMetadata(cluster) - // Adds the additional information - if err := sema.setInformation(&information{MachineName: machine.Name}); err != nil { - log.Error(err, "Failed to acquire init lock while setting semaphore information") - return false - } - - log.Info("Attempting to acquire the lock") - err = c.client.Create(ctx, sema.ConfigMap) - switch { - case apierrors.IsAlreadyExists(err): - log.Info("Cannot acquire the init lock. The init lock has been acquired by someone else") - return false - case err != nil: - log.Error(err, "Error acquiring the init lock") - return false - default: - return true - } -} - -// Unlock releases the lock. -func (c *ControlPlaneInitMutex) Unlock(ctx context.Context, cluster *clusterv1.Cluster) bool { - sema := newSemaphore() - cmName := configMapName(cluster.Name) - log := ctrl.LoggerFrom(ctx, "ConfigMap", klog.KRef(cluster.Namespace, cmName)) - err := c.client.Get(ctx, client.ObjectKey{ - Namespace: cluster.Namespace, - Name: cmName, - }, sema.ConfigMap) - switch { - case apierrors.IsNotFound(err): - log.Info("Control plane init lock not found, it may have been released already") - return true - case err != nil: - log.Error(err, "Error unlocking the control plane init lock") - return false - default: - // Delete the config map semaphore if there is no error fetching it - if err := c.client.Delete(ctx, sema.ConfigMap); err != nil { - if apierrors.IsNotFound(err) { - return true - } - log.Error(err, "Error deleting the config map underlying the control plane init lock") - return false - } - return true - } -} - -type information struct { - MachineName string `json:"machineName"` -} - -type semaphore struct { - *corev1.ConfigMap -} - -func newSemaphore() *semaphore { - return &semaphore{&corev1.ConfigMap{}} -} - -func configMapName(clusterName string) string { - return fmt.Sprintf("%s-lock", clusterName) -} - -func (s semaphore) information() (*information, error) { - li := &information{} - if err := json.Unmarshal([]byte(s.Data[semaphoreInformationKey]), li); err != nil { - return nil, errors.Wrap(err, "failed to unmarshal semaphore information") - } - return li, nil -} - -func (s semaphore) setInformation(information *information) error { - b, err := json.Marshal(information) - if err != nil { - return errors.Wrap(err, "failed to marshal semaphore information") - } - s.Data = map[string]string{} - s.Data[semaphoreInformationKey] = string(b) - return nil -} - -func (s *semaphore) setMetadata(cluster *clusterv1.Cluster) { - s.ObjectMeta = metav1.ObjectMeta{ - Namespace: cluster.Namespace, - Name: configMapName(cluster.Name), - Labels: map[string]string{ - clusterv1.ClusterLabelName: cluster.Name, - }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: cluster.APIVersion, - Kind: cluster.Kind, - Name: cluster.Name, - UID: cluster.UID, - }, - }, - } -} diff --git a/bootstrap/k3s/pkg/locking/control_plane_init_mutex_test.go b/bootstrap/k3s/pkg/locking/control_plane_init_mutex_test.go deleted file mode 100644 index f0d7d665d..000000000 --- a/bootstrap/k3s/pkg/locking/control_plane_init_mutex_test.go +++ /dev/null @@ -1,308 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package locking - -import ( - "context" - "errors" - "fmt" - "testing" - - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -const ( - clusterName = "test-cluster" - clusterNamespace = "test-namespace" -) - -var ( - ctx = ctrl.SetupSignalHandler() -) - -func TestControlPlaneInitMutex_Lock(t *testing.T) { - g := NewWithT(t) - - scheme := runtime.NewScheme() - g.Expect(clusterv1.AddToScheme(scheme)).To(Succeed()) - g.Expect(corev1.AddToScheme(scheme)).To(Succeed()) - - uid := types.UID("test-uid") - tests := []struct { - name string - client client.Client - shouldAcquire bool - }{ - { - name: "should successfully acquire lock if the config cannot be found", - client: &fakeClient{ - Client: fake.NewClientBuilder().WithScheme(scheme).Build(), - getError: apierrors.NewNotFound(schema.GroupResource{Group: "", Resource: "configmaps"}, fmt.Sprintf("%s-controlplane", uid)), - }, - shouldAcquire: true, - }, - { - name: "should not acquire lock if already exits", - client: &fakeClient{ - Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(&corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: configMapName(clusterName), - Namespace: clusterNamespace, - }, - }).Build(), - }, - shouldAcquire: false, - }, - { - name: "should not acquire lock if cannot create config map", - client: &fakeClient{ - Client: fake.NewClientBuilder().WithScheme(scheme).Build(), - getError: apierrors.NewNotFound(schema.GroupResource{Group: "", Resource: "configmaps"}, configMapName(clusterName)), - createError: errors.New("create error"), - }, - shouldAcquire: false, - }, - { - name: "should not acquire lock if config map already exists while creating", - client: &fakeClient{ - Client: fake.NewClientBuilder().WithScheme(scheme).Build(), - getError: apierrors.NewNotFound(schema.GroupResource{Group: "", Resource: "configmaps"}, fmt.Sprintf("%s-controlplane", uid)), - createError: apierrors.NewAlreadyExists(schema.GroupResource{Group: "", Resource: "configmaps"}, fmt.Sprintf("%s-controlplane", uid)), - }, - shouldAcquire: false, - }, - } - - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - gs := NewWithT(t) - - l := &ControlPlaneInitMutex{ - client: tc.client, - } - - cluster := &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: clusterNamespace, - Name: clusterName, - UID: uid, - }, - } - machine := &clusterv1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("machine-%s", cluster.Name), - }, - } - - gs.Expect(l.Lock(ctx, cluster, machine)).To(Equal(tc.shouldAcquire)) - }) - } -} - -func TestControlPlaneInitMutex_LockWithMachineDeletion(t *testing.T) { - g := NewWithT(t) - - scheme := runtime.NewScheme() - g.Expect(clusterv1.AddToScheme(scheme)).To(Succeed()) - g.Expect(corev1.AddToScheme(scheme)).To(Succeed()) - - newMachineName := "new-machine" - tests := []struct { - name string - client client.Client - expectedMachineName string - }{ - { - name: "should not give the lock to new machine if the machine that created it does exist", - client: &fakeClient{ - Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects( - &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: configMapName(clusterName), - Namespace: clusterNamespace}, - Data: map[string]string{ - "lock-information": "{\"machineName\":\"existent-machine\"}", - }}, - &clusterv1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Name: "existent-machine", - Namespace: clusterNamespace, - }, - }, - ).Build(), - }, - expectedMachineName: "existent-machine", - }, - { - name: "should give the lock to new machine if the machine that created it does not exist", - client: &fakeClient{ - Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects( - &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: configMapName(clusterName), - Namespace: clusterNamespace}, - Data: map[string]string{ - "lock-information": "{\"machineName\":\"non-existent-machine\"}", - }}, - ).Build(), - }, - expectedMachineName: newMachineName, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - l := &ControlPlaneInitMutex{ - client: tc.client, - } - - cluster := &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: clusterNamespace, - Name: clusterName, - }, - } - machine := &clusterv1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Name: newMachineName, - }, - } - - g.Eventually(func(g Gomega) error { - l.Lock(ctx, cluster, machine) - - cm := &corev1.ConfigMap{} - g.Expect(tc.client.Get(ctx, client.ObjectKey{ - Name: configMapName(clusterName), - Namespace: cluster.Namespace, - }, cm)).To(Succeed()) - - info, err := semaphore{cm}.information() - g.Expect(err).To(BeNil()) - - g.Expect(info.MachineName).To(Equal(tc.expectedMachineName)) - return nil - }, "20s").Should(Succeed()) - }) - } -} - -func TestControlPlaneInitMutex_UnLock(t *testing.T) { - uid := types.UID("test-uid") - configMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: configMapName(clusterName), - Namespace: clusterNamespace, - }, - } - tests := []struct { - name string - client client.Client - shouldRelease bool - }{ - { - name: "should release lock by deleting config map", - client: &fakeClient{ - Client: fake.NewClientBuilder().Build(), - }, - shouldRelease: true, - }, - { - name: "should not release lock if cannot delete config map", - client: &fakeClient{ - Client: fake.NewClientBuilder().WithObjects(configMap).Build(), - deleteError: errors.New("delete error"), - }, - shouldRelease: false, - }, - { - name: "should release lock if config map does not exist", - client: &fakeClient{ - Client: fake.NewClientBuilder().Build(), - getError: apierrors.NewNotFound(schema.GroupResource{Group: "", Resource: "configmaps"}, fmt.Sprintf("%s-controlplane", uid)), - }, - shouldRelease: true, - }, - { - name: "should not release lock if error while getting config map", - client: &fakeClient{ - Client: fake.NewClientBuilder().Build(), - getError: errors.New("get error"), - }, - shouldRelease: false, - }, - } - - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - gs := NewWithT(t) - - l := &ControlPlaneInitMutex{ - client: tc.client, - } - - cluster := &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: clusterNamespace, - Name: clusterName, - UID: uid, - }, - } - - gs.Expect(l.Unlock(ctx, cluster)).To(Equal(tc.shouldRelease)) - }) - } -} - -type fakeClient struct { - client.Client - getError error - createError error - deleteError error -} - -func (fc *fakeClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error { - if fc.getError != nil { - return fc.getError - } - return fc.Client.Get(ctx, key, obj) -} - -func (fc *fakeClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { - if fc.createError != nil { - return fc.createError - } - return fc.Client.Create(ctx, obj, opts...) -} - -func (fc *fakeClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { - if fc.deleteError != nil { - return fc.deleteError - } - return fc.Client.Delete(ctx, obj, opts...) -} diff --git a/bootstrap/k3s/pkg/types/config.go b/bootstrap/k3s/pkg/types/config.go deleted file mode 100644 index f6c4f6525..000000000 --- a/bootstrap/k3s/pkg/types/config.go +++ /dev/null @@ -1,131 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package types - -// DefaultK3sConfigLocation is the default location for the k3s config file. -const DefaultK3sConfigLocation = "/etc/rancher/k3s/config.yaml" - -// K3sServerConfiguration is the configuration for the k3s server. -type K3sServerConfiguration struct { - // Database - DataStoreEndPoint string `json:"datastore-endpoint,omitempty"` - DataStoreCAFile string `json:"datastore-cafile,omitempty"` - DataStoreCertFile string `json:"datastore-certfile,omitempty"` - DataStoreKeyFile string `json:"datastore-keyfile,omitempty"` - - // Cluster - Token string `json:"token,omitempty"` - TokenFile string `json:"token-file,omitempty"` - Server string `json:"server,omitempty"` - ClusterInit bool `json:"cluster-init,omitempty"` - - // Listener - // BindAddress k3s bind address. - BindAddress string `json:"bind-address,omitempty"` - // HTTPSListenPort HTTPS listen port. - HTTPSListenPort int `json:"https-listen-port,omitempty"` - // AdvertiseAddress IP address that apiserver uses to advertise to members of the cluster. - AdvertiseAddress string `json:"advertise-address,omitempty"` - // AdvertisePort Port that apiserver uses to advertise to members of the cluster (default: listen-port). - AdvertisePort int `json:"advertise-port,omitempty"` - // TLSSan Add additional hostname or IP as a Subject Alternative Name in the TLS cert. - TLSSan string `json:"tls-san,omitempty"` - - // Networking - // ClusterCIDR Network CIDR to use for pod IPs. - ClusterCIDR string `json:"cluster-cidr,omitempty"` - // ServiceCIDR Network CIDR to use for services IPs. - ServiceCIDR string `json:"service-cidr,omitempty"` - // ServiceNodePortRange Port range to reserve for services with NodePort visibility. - ServiceNodePortRange string `json:"service-node-port-range,omitempty"` - // ClusterDNS cluster IP for coredns service. Should be in your service-cidr range. - ClusterDNS string `json:"cluster-dns,omitempty"` - // ClusterDomain cluster Domain. - ClusterDomain string `json:"cluster-domain,omitempty"` - // FlannelBackend One of ‘none’, ‘vxlan’, ‘ipsec’, ‘host-gw’, or ‘wireguard’. (default: vxlan) - FlannelBackend string `json:"flannel-backend,omitempty"` - - // Kubernetes components - // Disable do not deploy packaged components and delete any deployed components - // (valid items: coredns, servicelb, traefik,local-storage, metrics-server). - Disable string `json:"disable,omitempty"` - // DisableKubeProxy disable running kube-proxy. - DisableKubeProxy bool `json:"disable-kube-roxy,omitempty"` - // DisableNetworkPolicy disable k3s default network policy controller. - DisableNetworkPolicy bool `json:"disable-network-policy,omitempty"` - // DisableHelmController disable Helm controller. - DisableHelmController bool `json:"disable-helm-controller,omitempty"` - - // Kubernetes processes - // DisableCloudController Disable k3s default cloud controller manager. - DisableCloudController bool `json:"disable-cloud-controller,omitempty"` - // KubeAPIServerArgs Customized flag for kube-apiserver process. - KubeAPIServerArgs []string `json:"kube-apiserver-arg,omitempty"` - // KubeControllerManagerArgs Customized flag for kube-controller-manager process. - KubeControllerManagerArgs []string `json:"kube-controller-manager-arg,omitempty"` - // KubeSchedulerArgs Customized flag for kube-scheduler process. - KubeSchedulerArgs []string `json:"kube-scheduler-args,omitempty"` - - // Agent - K3sAgentConfiguration `json:",inline"` -} - -// K3sAgentConfiguration is the configuration for the k3s agent. -type K3sAgentConfiguration struct { - // Cluster - Token string `json:"token,omitempty"` - TokenFile string `json:"token-file,omitempty"` - Server string `json:"server,omitempty"` - - // NodeName k3s node name. - NodeName string `json:"node-name,omitempty"` - // NodeLabels registering and starting kubelet with set of labels. - NodeLabels []string `json:"node-label,omitempty"` - // NodeTaints registering and starting kubelet with set of taints. - NodeTaints []string `json:"node-taint,omitempty"` - // SeLinux Enable SELinux in containerd - SeLinux bool `json:"selinux,omitempty"` - // LBServerPort - // Local port for supervisor client load-balancer. - // If the supervisor and apiserver are not colocated an additional port 1 less than this port - // will also be used for the apiserver client load-balancer. (default: 6444) - LBServerPort int `json:"lb-server-port,omitempty"` - // DataDir Folder to hold state. - DataDir string `json:"data-dir,omitempty"` - - // Runtime - // ContainerRuntimeEndpoint Disable embedded containerd and use alternative CRI implementation. - ContainerRuntimeEndpoint string `json:"container-runtime-endpoint,omitempty"` - // PauseImage Customized pause image for containerd or Docker sandbox. - PauseImage string `json:"pause-image,omitempty"` - // PrivateRegistry Path to a private registry configuration file. - PrivateRegistry string `json:"private-registry,omitempty"` - - // Networking - // NodeIP IP address to advertise for node. - NodeIP string `json:"node-ip,omitempty"` - // NodeExternalIP External IP address to advertise for node. - NodeExternalIP string `json:"node-external-ip,omitempty"` - // ResolvConf Path to Kubelet resolv.conf file. - ResolvConf string `json:"resolv-conf,omitempty"` - - // Kubernetes - // KubeletArgs Customized flag for kubelet process. - KubeletArgs []string `json:"kubelet-arg,omitempty"` - // KubeProxyArgs Customized flag for kube-proxy process. - KubeProxyArgs []string `json:"kube-proxy-arg,omitempty"` -} diff --git a/bootstrap/k3s/pkg/types/doc.go b/bootstrap/k3s/pkg/types/doc.go deleted file mode 100644 index 7a55aa89f..000000000 --- a/bootstrap/k3s/pkg/types/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package types contains k3s config types. -package types diff --git a/bootstrap/k3s/pkg/types/util.go b/bootstrap/k3s/pkg/types/util.go deleted file mode 100644 index f57910447..000000000 --- a/bootstrap/k3s/pkg/types/util.go +++ /dev/null @@ -1,177 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package types - -import ( - "fmt" - "strings" - - "github.com/jinzhu/copier" - kubeyaml "sigs.k8s.io/yaml" - - infrabootstrapv1 "github.com/kubesphere/kubekey/v3/bootstrap/k3s/api/v1beta1" -) - -// MarshalInitServerConfiguration marshals the ServerConfiguration object into a string. -func MarshalInitServerConfiguration(spec *infrabootstrapv1.K3sConfigSpec, token string) (string, error) { - obj := spec.ServerConfiguration - serverConfig := &K3sServerConfiguration{} - if err := copier.Copy(serverConfig, obj.Database); err != nil { - return "", err - } - if err := copier.Copy(serverConfig, obj.Listener); err != nil { - return "", err - } - if err := copier.Copy(serverConfig, obj.Networking); err != nil { - return "", err - } - if err := copier.Copy(serverConfig, obj.KubernetesComponents); err != nil { - return "", err - } - - serverConfig.Token = token - serverConfig.ClusterInit = *obj.Database.ClusterInit - - serverConfig.DisableCloudController = true - serverConfig.KubeAPIServerArgs = append(obj.KubernetesProcesses.KubeAPIServerArgs, "anonymous-auth=true", getTLSCipherSuiteArg()) - serverConfig.KubeControllerManagerArgs = append(obj.KubernetesProcesses.KubeControllerManagerArgs, "cloud-provider=external") - serverConfig.KubeSchedulerArgs = obj.KubernetesProcesses.KubeSchedulerArgs - - serverConfig.K3sAgentConfiguration = K3sAgentConfiguration{ - NodeName: obj.Agent.Node.NodeName, - NodeLabels: obj.Agent.Node.NodeLabels, - NodeTaints: obj.Agent.Node.NodeTaints, - SeLinux: obj.Agent.Node.SeLinux, - LBServerPort: obj.Agent.Node.LBServerPort, - DataDir: obj.Agent.Node.DataDir, - ContainerRuntimeEndpoint: obj.Agent.Runtime.ContainerRuntimeEndpoint, - PauseImage: obj.Agent.Runtime.PauseImage, - PrivateRegistry: obj.Agent.Runtime.PrivateRegistry, - NodeIP: obj.Agent.Networking.NodeIP, - NodeExternalIP: obj.Agent.Networking.NodeExternalIP, - ResolvConf: obj.Agent.Networking.ResolvConf, - KubeletArgs: obj.Agent.KubernetesAgentProcesses.KubeletArgs, - KubeProxyArgs: obj.Agent.KubernetesAgentProcesses.KubeProxyArgs, - } - - b, err := kubeyaml.Marshal(serverConfig) - if err != nil { - return "", err - } - return string(b), nil -} - -// MarshalJoinServerConfiguration marshals the join ServerConfiguration object into a string. -func MarshalJoinServerConfiguration(spec *infrabootstrapv1.K3sConfigSpec) (string, error) { - obj := spec.ServerConfiguration - serverConfig := &K3sServerConfiguration{} - if err := copier.Copy(serverConfig, obj.Database); err != nil { - return "", err - } - if err := copier.Copy(serverConfig, obj.Listener); err != nil { - return "", err - } - if err := copier.Copy(serverConfig, obj.Networking); err != nil { - return "", err - } - if err := copier.Copy(serverConfig, obj.KubernetesComponents); err != nil { - return "", err - } - - serverConfig.TokenFile = spec.Cluster.TokenFile - serverConfig.Token = spec.Cluster.Token - serverConfig.Server = spec.Cluster.Server - - serverConfig.DisableCloudController = true - serverConfig.KubeAPIServerArgs = append(obj.KubernetesProcesses.KubeAPIServerArgs, "anonymous-auth=true", getTLSCipherSuiteArg()) - serverConfig.KubeControllerManagerArgs = append(obj.KubernetesProcesses.KubeControllerManagerArgs, "cloud-provider=external") - serverConfig.KubeSchedulerArgs = obj.KubernetesProcesses.KubeSchedulerArgs - - serverConfig.K3sAgentConfiguration = K3sAgentConfiguration{ - NodeName: obj.Agent.Node.NodeName, - NodeLabels: obj.Agent.Node.NodeLabels, - NodeTaints: obj.Agent.Node.NodeTaints, - SeLinux: obj.Agent.Node.SeLinux, - LBServerPort: obj.Agent.Node.LBServerPort, - DataDir: obj.Agent.Node.DataDir, - ContainerRuntimeEndpoint: obj.Agent.Runtime.ContainerRuntimeEndpoint, - PauseImage: obj.Agent.Runtime.PauseImage, - PrivateRegistry: obj.Agent.Runtime.PrivateRegistry, - NodeIP: obj.Agent.Networking.NodeIP, - NodeExternalIP: obj.Agent.Networking.NodeExternalIP, - ResolvConf: obj.Agent.Networking.ResolvConf, - KubeletArgs: obj.Agent.KubernetesAgentProcesses.KubeletArgs, - KubeProxyArgs: obj.Agent.KubernetesAgentProcesses.KubeProxyArgs, - } - - b, err := kubeyaml.Marshal(serverConfig) - if err != nil { - return "", err - } - return string(b), nil -} - -// MarshalJoinAgentConfiguration marshals the join AgentConfiguration object into a string. -func MarshalJoinAgentConfiguration(spec *infrabootstrapv1.K3sConfigSpec) (string, error) { - obj := spec.AgentConfiguration - agentConfig := &K3sAgentConfiguration{} - if err := copier.Copy(agentConfig, obj.Node); err != nil { - return "", err - } - if err := copier.Copy(agentConfig, obj.Networking); err != nil { - return "", err - } - if err := copier.Copy(agentConfig, obj.Runtime); err != nil { - return "", err - } - if err := copier.Copy(agentConfig, obj.KubernetesAgentProcesses); err != nil { - return "", err - } - - agentConfig.TokenFile = spec.Cluster.TokenFile - agentConfig.Token = spec.Cluster.Token - agentConfig.Server = spec.Cluster.Server - - b, err := kubeyaml.Marshal(agentConfig) - if err != nil { - return "", err - } - return string(b), nil -} - -func getTLSCipherSuiteArg() string { - ciphers := []string{ - // Modern Compatibility recommended configuration in - // https://wiki.mozilla.org/Security/Server_Side_TLS#Modern_compatibility - "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", - "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", - "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", - "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", - "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", - "TLS_RSA_WITH_AES_128_GCM_SHA256", - "TLS_RSA_WITH_AES_256_GCM_SHA384", - } - - ciphersList := "" - for _, cc := range ciphers { - ciphersList += cc + "," - } - ciphersList = strings.TrimRight(ciphersList, ",") - - return fmt.Sprintf("tls-cipher-suites=%s", ciphersList) -} diff --git a/build/controller-manager/Dockerfile b/build/controller-manager/Dockerfile new file mode 100644 index 000000000..3ca15dde0 --- /dev/null +++ b/build/controller-manager/Dockerfile @@ -0,0 +1,36 @@ +ARG builder_image +# Build the manager binary +FROM ${builder_image} as builder + +ARG goproxy=https://goproxy.cn,direct +ENV GOPROXY ${goproxy} + +WORKDIR /workspace + +COPY api/go.mod api/go.mod +COPY api/go.sum api/go.sum +COPY go.mod go.mod +COPY go.sum go.sum + +# Cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN --mount=type=cache,target=/go/pkg/mod go mod download + +# Copy the go source +COPY ./ ./ + +ARG ldflags +ARG build_tags + +# Cache the go build into the the Go’s compiler cache folder so we take benefits of compiler caching across docker build calls +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 go build -trimpath -tags "${build_tags}" -ldflags "${ldflags}" -o controller-manager cmd/controller-manager/controller_manager.go + +FROM alpine:3.19.0 + +WORKDIR /kubekey + +COPY --from=builder /workspace/controller-manager /usr/local/bin/controller-manager + +ENTRYPOINT ["sh","-c"] diff --git a/build/kk/Dockerfile b/build/kk/Dockerfile new file mode 100644 index 000000000..6e6f5bbc4 --- /dev/null +++ b/build/kk/Dockerfile @@ -0,0 +1,45 @@ +ARG builder_image +# Build the manager binary +FROM ${builder_image} as builder + +ARG goproxy=https://goproxy.cn,direct +ENV GOPROXY ${goproxy} + +WORKDIR /workspace + +COPY api/go.mod api/go.mod +COPY api/go.sum api/go.sum +COPY go.mod go.mod +COPY go.sum go.sum + +# Cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN --mount=type=cache,target=/go/pkg/mod go mod download + +# Copy the go source +COPY ./ ./ + +ARG ldflags +ARG build_tags + +# Cache the go build into the the Go’s compiler cache folder so we take benefits of compiler caching across docker build calls +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 go build -trimpath -tags "${build_tags}" -ldflags "${ldflags}" -o kk cmd/kk/kubekey.go + +FROM alpine:3.19.0 + +WORKDIR /kubekey + +# install tool +RUN apk update && apk add bash && apk add curl && apk add openssl && apk add sudo +RUN curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 && \ + chmod 700 get_helm.sh && \ + ./get_helm.sh + +COPY --from=ghcr.io/oras-project/oras:v1.1.0 /bin/oras /usr/local/bin/oras +COPY --from=builder /workspace/kk /usr/local/bin/kk +# add builtin capkk project +COPY --from=builder /workspace/builtin/capkk /capkk/project + +ENTRYPOINT ["sh"] diff --git a/builtin/capkk/playbooks/add_node.yaml b/builtin/capkk/playbooks/add_node.yaml new file mode 100644 index 000000000..e32faaa59 --- /dev/null +++ b/builtin/capkk/playbooks/add_node.yaml @@ -0,0 +1,76 @@ +--- +- hosts: + - "{{ .node_name }}" + gather_facts: true + vars: + architectures: + amd64: + - amd64 + - x86_64 + arm64: + - arm64 + - aarch64 + tasks: + - name: Get os arch for each node + set_fact: + binary_type: >- + {{- if .architectures.amd64 | has .os.architecture -}} + amd64 + {{- else if .architectures.arm64 | has .os.architecture -}} + arm64 + {{- end -}} + +- hosts: + - "{{ .node_name }}" + gather_facts: true + vars_files: + - vars/main.yaml + roles: + - precheck/env_check + +- name: init cluster + hosts: + - localhost + vars_files: + - vars/main.yaml + tasks: + # https://cluster-api.sigs.k8s.io/tasks/bootstrap/kubeadm-bootstrap/index.html?highlight=ntp#additional-features + - name: init cloud-config + when: .kubernetes_installed | default false | eq false + block: + - name: get cloud-config value + command: | + cat {{ .cloud_config_dir }}/cloud-config/value + register: cloud_config_out + register_type: yaml + - name: add cloud_config to all hosts + add_hostvars: + hosts: all + vars: + cloud_config: >- + {{ .hostVars.localhost.cloud_config_out.stdout }} + roles: + - role: init/init-artifacts + when: .kubernetes_installed | default false | eq false + +- hosts: + - "{{ .node_name }}" + roles: + - role: init/init-os + when: .kubernetes_installed | default false | eq false + +- name: install cluster + hosts: + - "{{ .node_name }}" + gather_facts: true + roles: + - role: install/cri + when: .kubernetes_installed | default false | eq false + - role: install/kubernetes + when: .kubernetes_installed | default false | eq false + - role: install/cloud-config + when: .kubernetes_installed | default false | eq false + - role: install/cni + when: .kubernetes_installed | default false | eq false + - role: install/storageclass + when: .kubernetes_installed | default false | eq false \ No newline at end of file diff --git a/builtin/capkk/playbooks/delete_node.yaml b/builtin/capkk/playbooks/delete_node.yaml new file mode 100644 index 000000000..3e1d774e8 --- /dev/null +++ b/builtin/capkk/playbooks/delete_node.yaml @@ -0,0 +1,11 @@ +--- +- name: uninstall node + hosts: + - "{{ .node_name }}" + vars_files: + - vars/main.yaml + gather_facts: true + roles: + - uninstall/kubernetes + - uninstall/cri + - uninstall/cloud-config diff --git a/builtin/capkk/playbooks/host_check.yaml b/builtin/capkk/playbooks/host_check.yaml new file mode 100644 index 000000000..7e84bc8e3 --- /dev/null +++ b/builtin/capkk/playbooks/host_check.yaml @@ -0,0 +1,8 @@ +--- +- name: Check Connect + hosts: + - | + {{ .check_group | default "ungrouped" }} + tasks: + - name: Check Connect for Hosts + command: echo success \ No newline at end of file diff --git a/builtin/capkk/playbooks/vars/main.yaml b/builtin/capkk/playbooks/vars/main.yaml new file mode 100644 index 000000000..960c1b49a --- /dev/null +++ b/builtin/capkk/playbooks/vars/main.yaml @@ -0,0 +1,36 @@ +# the pod's WORKDIR which set by image. store the runtime files. +work_dir: /kubekey +# binary_dir may mount by playbook. usage it shouldn't be changed. +binary_dir: /capkk/kubekey +# cloud_config_dir may mount by playbook. usage it shouldn't be changed. +cloud_config_dir: /capkk/cloud +# tmp_dir for kubekey in remote node. it will store file like binary package, iso file etc. +tmp_dir: /tmp/kubekey + +# global_registry for all image +global_registry: "" +# dockerio_registry for docker.io image. +dockerio_registry: >- + {{- if .global_registry | empty | not -}} + {{ .global_registry }} + {{- else -}} + docker.io + {{- end -}} +# quayio_registry for quay.io image. +quayio_registry: >- + {{- if .global_registry | empty | not -}} + {{ .global_registry }} + {{- else -}} + quay.io + {{- end -}} +# ghcrio_registry for ghcr.io image. +ghcrio_registry: >- + {{- if .global_registry | empty | not -}} + {{ .global_registry }} + {{- else -}} + ghcr.io + {{- end -}} + +cri: + # support: containerd,docker + container_manager: docker \ No newline at end of file diff --git a/builtin/capkk/roles/init/init-artifacts/defaults/main.yaml b/builtin/capkk/roles/init/init-artifacts/defaults/main.yaml new file mode 100644 index 000000000..79a061f4c --- /dev/null +++ b/builtin/capkk/roles/init/init-artifacts/defaults/main.yaml @@ -0,0 +1,185 @@ +artifact: + arch: [ "amd64" ] + # offline artifact package for kk. + artifact_file: "" + # the md5_file of artifact_file. + artifact_md5: "" + artifact_url: + etcd: + amd64: >- + {{- if .kkzone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/etcd/release/download/{{ .etcd_version }}/etcd-{{ .etcd_version }}-linux-amd64.tar.gz + {{- else -}} + https://github.com/etcd-io/etcd/releases/download/{{ .etcd_version }}/etcd-{{ .etcd_version }}-linux-amd64.tar.gz + {{- end -}} + arm64: >- + {{- if .kkzone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/etcd/release/download/{{ .etcd_version }}/etcd-{{ .etcd_version }}-linux-arm64.tar.gz + {{- else -}} + https://github.com/etcd-io/etcd/releases/download/{{ .etcd_version }}/etcd-{{ .etcd_version }}-linux-arm64.tar.gz + {{- end -}} + kubeadm: + amd64: >- + {{- if .kkzone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/release/{{ .kube_version }}/bin/linux/amd64/kubeadm + {{- else -}} + https://storage.googleapis.com/kubernetes-release/release/{{ .kube_version }}/bin/linux/amd64/kubeadm + {{- end -}} + arm64: >- + {{- if .kkzone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/release/{{ .kube_version }}/bin/linux/arm64/kubeadm + {{- else -}} + https://storage.googleapis.com/kubernetes-release/release/{{ .kube_version }}/bin/linux/arm64/kubeadm + {{- end -}} + kubelet: + amd64: >- + {{- if .kkzone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/release/{{ .kube_version }}/bin/linux/amd64/kubelet + {{- else -}} + https://storage.googleapis.com/kubernetes-release/release/{{ .kube_version }}/bin/linux/amd64/kubelet + {{- end -}} + arm64: >- + {{- if .kkzone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/release/{{ .kube_version }}/bin/linux/arm64/kubelet + {{- else -}} + https://storage.googleapis.com/kubernetes-release/release/{{ .kube_version }}/bin/linux/arm64/kubelet + {{- end -}} + kubectl: + amd64: >- + {{- if .kkzone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/release/{{ .kube_version }}/bin/linux/amd64/kubectl + {{- else -}} + https://storage.googleapis.com/kubernetes-release/release/{{ .kube_version }}/bin/linux/amd64/kubectl + {{- end -}} + arm64: >- + {{- if .kkzone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/release/{{ .kube_version }}/bin/linux/arm64/kubectl + {{- else -}} + https://storage.googleapis.com/kubernetes-release/release/{{ .kube_version }}/bin/linux/arm64/kubectl + {{- end -}} + cni_plugins: + amd64: >- + {{- if .kkzone | eq "cn" -}} + https://github.com/containernetworking/plugins/releases/download/{{ .cni_plugins_version }}/cni-plugins-linux-amd64-{{ .cni_plugins_version }}.tgz + {{- else -}} + https://containernetworking.pek3b.qingstor.com/plugins/releases/download/{{ .cni_plugins_version }}/cni-plugins-linux-amd64-{{ .cni_plugins_version }}.tgz + {{- end -}} + arm64: >- + {{- if .kkzone | eq "cn" -}} + https://github.com/containernetworking/plugins/releases/download/{{ .cni_plugins_version }}/cni-plugins-linux-arm64-{{ .cni_plugins_version }}.tgz + {{- else -}} + https://containernetworking.pek3b.qingstor.com/plugins/releases/download/{{ .cni_plugins_version }}/cni-plugins-linux-arm64-{{ .cni_plugins_version }}.tgz + {{- end -}} + helm: + amd64: >- + {{- if .kkzone | eq "cn" -}} + https://kubernetes-helm.pek3b.qingstor.com/helm-{{ .helm_version }}-linux-amd64.tar.gz + {{- else -}} + https://get.helm.sh/helm-{{ .helm_version }}-linux-amd64.tar.gz + {{- end -}} + arm64: >- + {{- if .kkzone | eq "cn" -}} + https://kubernetes-helm.pek3b.qingstor.com/helm-{{ .helm_version }}-linux-arm64.tar.gz + {{- else -}} + https://get.helm.sh/helm-{{ .helm_version }}-linux-arm64.tar.gz + {{- end -}} + crictl: + amd64: >- + {{- if .kkzone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/cri-tools/releases/download/{{ .crictl_version }}/crictl-{{ .crictl_version }}-linux-amd64.tar.gz + {{- else -}} + https://github.com/kubernetes-sigs/cri-tools/releases/download/{{ .crictl_version }}/crictl-{{ .crictl_version }}-linux-amd64.tar.gz + {{- end -}} + arm64: >- + {{- if .kkzone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/cri-tools/releases/download/{{ .crictl_version }}/crictl-{{ .crictl_version }}-linux-arm64.tar.gz + {{- else -}} + https://github.com/kubernetes-sigs/cri-tools/releases/download/{{ .crictl_version }}/crictl-{{ .crictl_version }}-linux-arm64.tar.gz + {{- end -}} + docker: + amd64: >- + {{- if .kkzone | eq "cn" -}} + https://mirrors.aliyun.com/docker-ce/linux/static/stable/x86_64/docker-{{ .docker_version }}.tgz + {{- else -}} + https://download.docker.com/linux/static/stable/x86_64/docker-{{ .docker_version }}.tgz + {{- end -}} + arm64: >- + {{- if .kkzone | eq "cn" -}} + https://mirrors.aliyun.com/docker-ce/linux/static/stable/aarch64/docker-{{ .docker_version }}.tgz + {{- else -}} + https://download.docker.com/linux/static/stable/aarch64/docker-{{ .docker_version }}.tgz + {{- end -}} + cridockerd: + amd64: >- + {{- if .kkzone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/releases/download/{{ .cridockerd_version }}/cri-dockerd-{{ .cridockerd_version | default "" | trimPrefix "v" }}.amd64.tgz + {{- else -}} + https://github.com/Mirantis/cri-dockerd/releases/download/{{ .cridockerd_version }}/cri-dockerd-{{ .cridockerd_version | default "" | trimPrefix "v" }}.amd64.tgz + {{- end -}} + arm64: >- + {{- if .kkzone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/releases/download/{{ .cridockerd_version }}/cri-dockerd-{{ .cridockerd_version | default "" | trimPrefix "v" }}.arm64.tgz + {{- else -}} + https://github.com/Mirantis/cri-dockerd/releases/download/{{ .cridockerd_version }}/cri-dockerd-{{ .cridockerd_version | default "" | trimPrefix "v" }}.arm64.tgz + {{- end -}} + containerd: + amd64: >- + {{- if .kkzone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/containerd/containerd/releases/download/{{ .containerd_version }}/containerd-{{ .containerd_version | default "" | trimPrefix "v" }}-linux-amd64.tar.gz + {{- else -}} + https://github.com/containerd/containerd/releases/download/{{ .containerd_version }}/containerd-{{ .containerd_version | default "" | trimPrefix "v" }}-linux-amd64.tar.gz + {{- end -}} + arm64: >- + {{- if .kkzone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/containerd/containerd/releases/download/{{ .containerd_version }}/containerd-{{ .containerd_version | default "" | trimPrefix "v" }}-linux-arm64.tar.gz + {{- else -}} + https://github.com/containerd/containerd/releases/download/{{ .containerd_version }}/containerd-{{ .containerd_version | default "" | trimPrefix "v" }}-linux-arm64.tar.gz + {{- end -}} + runc: + amd64: >- + {{- if .kkzone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/opencontainers/runc/releases/download/{{ .runc_version }}/runc.amd64 + {{- else -}} + https://github.com/opencontainers/runc/releases/download/{{ .runc_version }}/runc.amd64 + {{- end -}} + arm64: >- + {{- if .kkzone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/opencontainers/runc/releases/download/{{ .runc_version }}/runc.arm64 + {{- else -}} + https://github.com/opencontainers/runc/releases/download/{{ .runc_version }}/runc.arm64 + {{- end -}} + dockercompose: + amd64: >- + {{- if .kkzone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/docker/compose/releases/download/{{ .dockercompose_version }}/docker-compose-linux-x86_64 + {{- else -}} + https://github.com/docker/compose/releases/download/{{ .dockercompose_version }}/docker-compose-linux-x86_64 + {{- end -}} + arm64: >- + {{- if .kkzone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/docker/compose/releases/download/{{ .dockercompose_version }}/docker-compose-linux-aarch64 + {{- else -}} + https://github.com/docker/compose/releases/download/{{ .dockercompose_version }}/docker-compose-linux-aarch64 + {{- end -}} + # Notice: In the early calico helm chart, appVersion is not same as version(eg. v3.17.4) + calico: https://github.com/projectcalico/calico/releases/download/{{ .calico_version }}/tigera-operator-{{ .calico_version }}.tgz + calicoctl: + amd64: >- + {{- if .kkzone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/projectcalico/calico/releases/download/{{ .calico_version }}/calicoctl-linux-amd64 + {{- else -}} + https://github.com/projectcalico/calico/releases/download/{{ .calico_version }}/calicoctl-linux-amd64 + {{- end -}} + arm64: >- + {{- if .kkzone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/projectcalico/calico/releases/download/{{ .calico_version }}/calicoctl-linux-arm64 + {{- else -}} + https://github.com/projectcalico/calico/releases/download/{{ .calico_version }}/calicoctl-linux-arm64 + {{- end -}} + flannel: https://github.com/flannel-io/flannel/releases/download/{{ .fannel_version }}/flannel.tgz + cilium: https://helm.cilium.io/cilium-{{ .cilium_version }}.tgz + ciliumcli: + amd64: >- + https://github.com/cilium/cilium-cli/releases/download/{{ .ciliumcli_version }}/cilium-linux-amd64.tar.gz + arm64: >- + https://github.com/cilium/cilium-cli/releases/download/{{ .ciliumcli_version }}/cilium-linux-arm64.tar.gz diff --git a/builtin/capkk/roles/init/init-artifacts/tasks/download_binary.yaml b/builtin/capkk/roles/init/init-artifacts/tasks/download_binary.yaml new file mode 100644 index 000000000..bec3e14d3 --- /dev/null +++ b/builtin/capkk/roles/init/init-artifacts/tasks/download_binary.yaml @@ -0,0 +1,191 @@ +--- +- name: Check binaries for kube + command: | + kube_path={{ .binary_dir }}/kube/{{ .kube_version }}/{{ .item }} + if [ ! -f $kube_path/kubelet ]; then + mkdir -p $kube_path + # download online + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.kubelet .item }}) + if [ $http_code != 200 ]; then + echo "http code is $http_code" + exit 1 + fi + curl -L -o $kube_path/kubelet {{ get .artifact.artifact_url.kubelet .item }} + fi + if [ ! -f $kube_path/kubeadm ]; then + mkdir -p $kube_path + # download online + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.kubeadm .item }}) + if [ $http_code != 200 ]; then + echo "http code is $http_code" + exit 1 + fi + curl -L -o $kube_path/kubeadm {{ get .artifact.artifact_url.kubeadm .item }} + fi + if [ ! -f $kube_path/kubectl ]; then + mkdir -p $kube_path + # download online + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.kubectl .item }}) + if [ $http_code != 200 ]; then + echo "http code is $http_code" + exit 1 + fi + curl -L -o $kube_path/kubectl {{ get .artifact.artifact_url.kubectl .item }} + fi + loop: "{{ .artifact.arch | toJson }}" + when: .kube_version | empty | not + +- name: Check binaries for cni_plugins + command: | + artifact_name={{ get .artifact.artifact_url.cni_plugins .item | splitList "/" | last }} + artifact_path={{ .binary_dir }}/cni/plugins/{{ .cni_plugins_version }}/{{ .item }} + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # download online + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.cni_plugins .item }}) + if [ $http_code != 200 ]; then + echo "http code is $http_code" + exit 1 + fi + curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.cni_plugins .item }} + fi + loop: "{{ .artifact.arch | toJson }}" + when: .cni_plugins_version | empty | not + +- name: Check binaries for ciliumcli + command: | + artifact_name={{ get .artifact.artifact_url.ciliumcli .item | splitList "/" | last }} + artifact_path={{ .binary_dir }}/cni/cilium/ciliumcli-{{ .ciliumcli_version }}/{{ .item }} + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # download online + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.helm .item }}) + if [ $http_code != 200 ]; then + echo "http code is $http_code" + exit 1 + fi + curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.helm .item }} + fi + loop: "{{ .artifact.arch | toJson }}" + when: + - .cilium_version | empty | not + - .ciliumcli_version | empty | not + +- name: Check binaries for helm + command: | + artifact_name={{ get .artifact.artifact_url.helm .item | splitList "/" | last }} + artifact_path={{ .binary_dir }}/helm/{{ .helm_version }}/{{ .item }} + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # download online + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.helm .item }}) + if [ $http_code != 200 ]; then + echo "http code is $http_code" + exit 1 + fi + curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.helm .item }} + fi + loop: "{{ .artifact.arch | toJson }}" + when: .helm_version | empty | not + +- name: Check binaries for crictl + command: | + artifact_name={{ get .artifact.artifact_url.crictl .item | splitList "/" | last }} + artifact_path={{ .binary_dir }}/crictl/{{ .crictl_version }}/{{ .item }} + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # download online + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.crictl .item }}) + if [ $http_code != 200 ]; then + echo "http code is $http_code" + exit 1 + fi + curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.crictl .item }} + fi + loop: "{{ .artifact.arch | toJson }}" + when: .crictl_version | empty | not + +- name: Check binaries for docker + command: | + artifact_name={{ get .artifact.artifact_url.docker .item | splitList "/" | last }} + artifact_path={{ .binary_dir }}/docker/{{ .docker_version }}/{{ .item }} + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # download online + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.docker .item }}) + if [ $http_code != 200 ]; then + echo "http code is $http_code" + exit 1 + fi + curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.docker .item }} + fi + loop: "{{ .artifact.arch | toJson }}" + when: .docker_version | empty | not + +- name: Check binaries for cridockerd + command: | + artifact_name={{ get .artifact.artifact_url.cridockerd .item | splitList "/" | last }} + artifact_path={{ .binary_dir }}/cri-dockerd/{{ .cridockerd_version }}/{{ .item }} + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # download online + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.cridockerd .item }}) + if [ $http_code != 200 ]; then + echo "http code is $http_code" + exit 1 + fi + curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.cridockerd .item }} + fi + loop: "{{ .artifact.arch | toJson }}" + when: .cridockerd_version | empty | not + +- name: Check binaries for containerd + command: | + artifact_name={{ get .artifact.artifact_url.containerd .item | splitList "/" | last }} + artifact_path={{ .binary_dir }}/containerd/{{ .containerd_version }}/{{ .item }} + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # download online + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.containerd .item }}) + if [ $http_code != 200 ]; then + echo "http code is $http_code" + exit 1 + fi + curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.containerd .item }} + fi + loop: "{{ .artifact.arch | toJson }}" + when: .containerd_version | empty | not + +- name: Check binaries for runc + command: | + artifact_name={{ get .artifact.artifact_url.runc .item | splitList "/" | last }} + artifact_path={{ .binary_dir }}/runc/{{ .runc_version }}/{{ .item }} + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # download online + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.runc .item }}) + if [ $http_code != 200 ]; then + echo "http code is $http_code" + exit 1 + fi + curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.runc .item }} + fi + loop: "{{ .artifact.arch | toJson }}" + when: .runc_version | empty | not + +- name: Check binaries for calicoctl + command: | + artifact_name=calicoctl + artifact_path={{ .binary_dir }}/cni/calico/{{ .calico_version }}/{{ .item }} + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # download online + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .artifact.artifact_url.calicoctl .item }}) + if [ $http_code != 200 ]; then + echo "http code is $http_code" + exit 1 + fi + curl -L -o $artifact_path/$artifact_name {{ get .artifact.artifact_url.calicoctl .item }} + fi + loop: "{{ .artifact.arch | toJson }}" + when: .calico_version | empty | not \ No newline at end of file diff --git a/builtin/capkk/roles/init/init-artifacts/tasks/download_helm.yaml b/builtin/capkk/roles/init/init-artifacts/tasks/download_helm.yaml new file mode 100644 index 000000000..3e68c9a12 --- /dev/null +++ b/builtin/capkk/roles/init/init-artifacts/tasks/download_helm.yaml @@ -0,0 +1,33 @@ +--- +- name: Check binaries for calico + command: | + artifact_name={{ .artifact.artifact_url.calico | splitList "/" | last }} + artifact_path={{ .binary_dir }}/cni/calico + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # download online + curl -Lo $artifact_path/$artifact_name {{ .artifact.artifact_url.calico }} + fi + when: .calico_version | empty | not + +- name: Check binaries for cilium + command: | + artifact_name={{ .artifact.artifact_url.cilium | splitList "/" | last }} + artifact_path={{ .binary_dir }}/cni/cilium + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # download online + curl -Lo $artifact_path/$artifact_name {{ .artifact.artifact_url.cilium }} + fi + when: .cilium_version | empty | not + +- name: Check binaries for flannel + command: | + artifact_name={{ .artifact.artifact_url.flannel | splitList "/" | last }} + artifact_path={{ .binary_dir }}/cni/flannel + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # download online + curl -Lo $artifact_path/$artifact_name {{ .artifact.artifact_url.flannel }} + fi + when: .flannel_version | empty | not \ No newline at end of file diff --git a/builtin/capkk/roles/init/init-artifacts/tasks/main.yaml b/builtin/capkk/roles/init/init-artifacts/tasks/main.yaml new file mode 100644 index 000000000..df5978554 --- /dev/null +++ b/builtin/capkk/roles/init/init-artifacts/tasks/main.yaml @@ -0,0 +1,11 @@ +--- +- name: Create binaries dir + command: | + mkdir -p {{ .binary_dir }} + +- name: Download binaries + block: + # the binaries which download binary + - include_tasks: download_binary.yaml + # the binaries which download helm + - include_tasks: download_helm.yaml diff --git a/builtin/capkk/roles/init/init-os/defaults/main.yaml b/builtin/capkk/roles/init/init-os/defaults/main.yaml new file mode 100644 index 000000000..16f29a26f --- /dev/null +++ b/builtin/capkk/roles/init/init-os/defaults/main.yaml @@ -0,0 +1,12 @@ +ntp: + servers: >- + {{- with .cloud_config.ntp.servers }} + {{ . | toJson }} + {{- else }} + [ "cn.pool.ntp.org" ] + {{- end }} + enabled: >- + {{ .cloud_config.ntp.enabled | default true }} +timezone: Asia/Shanghai +# set hostname by inventory_host's name which defined in inventory.yaml +set_hostname: true \ No newline at end of file diff --git a/builtin/capkk/roles/init/init-os/tasks/init_ntpserver.yaml b/builtin/capkk/roles/init/init-os/tasks/init_ntpserver.yaml new file mode 100644 index 000000000..323168825 --- /dev/null +++ b/builtin/capkk/roles/init/init-os/tasks/init_ntpserver.yaml @@ -0,0 +1,51 @@ +--- +- name: Configure ntp server + command: | + chronyConfigFile={{ if or (.os.release.ID | eq "ubuntu") (.os.release.ID_LIKE | eq "debian") }}"/etc/chrony/chrony.conf"{{ else }}"/etc/chrony.conf"{{ end }} + # clear old server + sed -i '/^server/d' $chronyConfigFile + sed -i 's/^pool /#pool /g' $chronyConfigFile + sed -i '/^allow/d' $chronyConfigFile + sed -i '/^local/d' $chronyConfigFile + # add base config + echo "allow 0.0.0.0/0" >> $chronyConfigFile + echo "allow ::/0" >> $chronyConfigFile + echo "local stratum 10" >> $chronyConfigFile + # add server config + {{- range $server := (.ntp.servers | fromJson) }} + {{- $internalIPv4 := "" }} + {{- $internalIPv6 := "" }} + {{- range $.hostvars }} + {{- if eq .hostname $server }} + {{- $internalIPv4 = .internal_ipv4 }} + {{- $internalIPv6 = .internal_ipv6 }} + {{- end }} + {{- end }} + # add ntp server: {{ $server }} + {{- if $internalIPv4 }} + grep -q '^server {{ $internalIPv4 }} iburst' $chronyConfigFile || sed '1a server {{ $internalIPv4 }} iburst' -i $chronyConfigFile + {{- end }} + {{- if $internalIPv6 }} + grep -q '^server {{ $internalIPv6 }} iburst' $chronyConfigFile || sed '1a server [{{ $internalIPv6 }}] iburst' -i $chronyConfigFile + {{- end }} + {{- if and ($internalIPv4 | empty) ($internalIPv6 | empty) }} + grep -q '^server {{ $server }} iburst' $chronyConfigFile || sed '1a server {{ $server }} iburst' -i $chronyConfigFile + {{- end }} + {{- end }} + when: + - .ntp.enabled + - .ntp.servers | fromJson | empty | not + +- name: Set timezone + command: | + timedatectl set-timezone {{ .timezone }} + timedatectl set-ntp {{ and .ntp.enabled (.ntp.servers | fromJson empty | not) }} + when: or (and .ntp.enabled (.ntp.servers | fromJson | empty | not)) (.timezone | empty | not) + +- name: Restart ntp server + command: | + {{- if or (.os.release.ID | eq "ubuntu") (.os.release.ID_LIKE | eq "debian") }} + systemctl restart chrony.service + {{- end }} + systemctl restart chronyd.service + when: or (and .ntp.enabled (.ntp.servers | fromJson | empty | not)) (.timezone | empty | not) diff --git a/builtin/capkk/roles/init/init-os/tasks/init_repository.yaml b/builtin/capkk/roles/init/init-os/tasks/init_repository.yaml new file mode 100644 index 000000000..650fd47f8 --- /dev/null +++ b/builtin/capkk/roles/init/init-os/tasks/init_repository.yaml @@ -0,0 +1,79 @@ +--- +- name: Sync repository + block: + - name: Sync repository file + ignore_errors: true + copy: + src: >- + {{ .binary_dir }}/repository/{{ .os.release.ID_LIKE }}-{{ .os.release.VERSION_ID }}-{{ .binary_type }}.iso + dest: >- + {{ .tmp_dir }}/repository.iso + - name: Mount iso file + command: | + if [ -f "{{ .tmp_dir }}/repository.iso" ]; then + mount -t iso9660 -o loop {{ .tmp_dir }}/repository.iso {{ .tmp_dir }}/iso + fi + rescue: + - name: Unmount iso file + command: | + if [ -f "{{ .tmp_dir }}/repository.iso" ]; then + umount {{ .tmp_dir }}/iso + fi + +- name: Init repository + block: + - name: Init debian repository + command: | + now=$(date +"%Y-%m-%d %H:%M:%S") + if [ -f "{{ .tmp_dir }}/repository.iso" ];then + # backup + mv /etc/apt/sources.list /etc/apt/sources.list.kubekey-$now.bak + mv /etc/apt/sources.list.d /etc/apt/sources.list.d.kubekey-$now.bak + mkdir -p /etc/apt/sources.list.d + # add repository + rm -rf /etc/apt/sources.list.d/* + echo 'deb [trusted=yes] file://{{ .tmp_dir }}/iso /' > /etc/apt/sources.list.d/kubekey.list + # update repository + apt-get update + # install + apt install -y socat conntrack ipset ebtables chrony ipvsadm + # reset repository + rm -rf /etc/apt/sources.list.d + mv /etc/apt/sources.list.kubekey.bak-$now /etc/apt/sources.list + mv /etc/apt/sources.list.d.kubekey.bak-$now /etc/apt/sources.list.d + else + apt-get update && apt install -y socat conntrack ipset ebtables chrony ipvsadm + fi + when: .os.release.ID_LIKE | eq "debian" + - name: Init rhel repository + command: | + now=$(date +"%Y-%m-%d %H:%M:%S") + if [ -f "{{ .tmp_dir }}/repository.iso" ];then + # backup + mv /etc/yum.repos.d /etc/yum.repos.d.kubekey-$now.bak + mkdir -p /etc/yum.repos.d + # add repository + rm -rf /etc/yum.repos.d/* + cat << EOF > /etc/yum.repos.d/CentOS-local.repo + [base-local] + name=rpms-local + + baseurl=file://{{ .tmp_dir }}/repository.iso + + enabled=1 + + gpgcheck=0 + + EOF + # update repository + yum clean all && yum makecache + # install + yum install -y openssl socat conntrack ipset ebtables chrony ipvsadm + # reset repository + rm -rf /etc/yum.repos.d + mv /etc/yum.repos.d.kubekey.bak-$now /etc/yum.repos.d + else + # install + yum install -y openssl socat conntrack ipset ebtables chrony ipvsadm + fi + when: .os.release.ID_LIKE | unquote | eq "rhel fedora" diff --git a/builtin/capkk/roles/init/init-os/tasks/main.yaml b/builtin/capkk/roles/init/init-os/tasks/main.yaml new file mode 100644 index 000000000..ded40bcf7 --- /dev/null +++ b/builtin/capkk/roles/init/init-os/tasks/main.yaml @@ -0,0 +1,23 @@ +--- +- include_tasks: init_repository.yaml + +- include_tasks: init_ntpserver.yaml + when: .ntp.enabled + +- name: Set hostname + command: | + hostnamectl set-hostname {{ .inventory_hostname }} \ + && sed -i '/^127.0.1.1/s/.*/127.0.1.1 {{ .inventory_hostname }}/g' /etc/hosts + when: + - .set_hostname + - .inventory_hostname | ne "localhost" + +- name: Sync init os to remote + template: + src: init-os.sh + dest: /etc/kubekey/scripts/init-os.sh + mode: 0755 + +- name: Execute init os script + command: | + /etc/kubekey/scripts/init-os.sh diff --git a/builtin/capkk/roles/init/init-os/templates/init-os.sh b/builtin/capkk/roles/init/init-os/templates/init-os.sh new file mode 100644 index 000000000..52b79b72b --- /dev/null +++ b/builtin/capkk/roles/init/init-os/templates/init-os.sh @@ -0,0 +1,268 @@ +#!/usr/bin/env bash + +# Copyright 2020 The KubeSphere Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------------------------ 1. Disable Swap and SELinux ----------------------- +swapoff -a +sed -i /^[^#]*swap*/s/^/\#/g /etc/fstab + +# See https://github.com/kubernetes/website/issues/14457 +if [ -f /etc/selinux/config ]; then + sed -ri 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config +fi +# for ubuntu: sudo apt install selinux-utils +# for centos: yum install selinux-policy +if command -v setenforce &> /dev/null +then + setenforce 0 + getenforce +fi + +# ------------------------ 2. Network Settings (Sysctl) ------------------------ + +echo 'net.core.netdev_max_backlog = 65535' >> /etc/sysctl.conf +echo 'net.core.rmem_max = 33554432' >> /etc/sysctl.conf +echo 'net.core.wmem_max = 33554432' >> /etc/sysctl.conf +echo 'net.core.somaxconn = 32768' >> /etc/sysctl.conf +echo 'net.bridge.bridge-nf-call-arptables = 1' >> /etc/sysctl.conf +echo 'vm.max_map_count = 262144' >> /etc/sysctl.conf +echo 'vm.swappiness = 0' >> /etc/sysctl.conf +echo 'vm.overcommit_memory = 1' >> /etc/sysctl.conf +echo 'fs.inotify.max_user_instances = 524288' >> /etc/sysctl.conf +echo 'fs.inotify.max_user_watches = 10240001' >> /etc/sysctl.conf +echo 'fs.pipe-max-size = 4194304' >> /etc/sysctl.conf +echo 'fs.aio-max-nr = 262144' >> /etc/sysctl.conf +echo 'kernel.pid_max = 65535' >> /etc/sysctl.conf +echo 'kernel.watchdog_thresh = 5' >> /etc/sysctl.conf +echo 'kernel.hung_task_timeout_secs = 5' >> /etc/sysctl.conf +{{- if .internal_ipv4 | empty | not }} +# add for ipv4 +echo 'net.ipv4.ip_forward = 1' >> /etc/sysctl.conf +echo 'net.bridge.bridge-nf-call-ip6tables = 1' >> /etc/sysctl.conf +echo 'net.ipv4.ip_local_reserved_ports = 30000-32767' >> /etc/sysctl.conf +echo 'net.ipv4.tcp_max_syn_backlog = 1048576' >> /etc/sysctl.conf +echo 'net.ipv4.neigh.default.gc_thresh1 = 512' >> /etc/sysctl.conf +echo 'net.ipv4.neigh.default.gc_thresh2 = 2048' >> /etc/sysctl.conf +echo 'net.ipv4.neigh.default.gc_thresh3 = 4096' >> /etc/sysctl.conf +echo 'net.ipv4.tcp_retries2 = 15' >> /etc/sysctl.conf +echo 'net.ipv4.tcp_max_tw_buckets = 1048576' >> /etc/sysctl.conf +echo 'net.ipv4.tcp_max_orphans = 65535' >> /etc/sysctl.conf +echo 'net.ipv4.udp_rmem_min = 131072' >> /etc/sysctl.conf +echo 'net.ipv4.udp_wmem_min = 131072' >> /etc/sysctl.conf +echo 'net.ipv4.conf.all.rp_filter = 1' >> /etc/sysctl.conf +echo 'net.ipv4.conf.default.rp_filter = 1' >> /etc/sysctl.conf +echo 'net.ipv4.conf.all.arp_accept = 1' >> /etc/sysctl.conf +echo 'net.ipv4.conf.default.arp_accept = 1' >> /etc/sysctl.conf +echo 'net.ipv4.conf.all.arp_ignore = 1' >> /etc/sysctl.conf +echo 'net.ipv4.conf.default.arp_ignore = 1' >> /etc/sysctl.conf +{{- end }} +{{- if .internal_ipv6 | empty | not }} +# add for ipv6 +echo 'net.bridge.bridge-nf-call-iptables = 1' >> /etc/sysctl.conf +echo 'net.ipv6.conf.all.disable_ipv6 = 0' >> /etc/sysctl.conf +echo 'net.ipv6.conf.default.disable_ipv6 = 0' >> /etc/sysctl.conf +echo 'net.ipv6.conf.lo.disable_ipv6 = 0' >> /etc/sysctl.conf +echo 'net.ipv6.conf.all.forwarding=1' >> /etc/sysctl.conf +echo 'net.ipv6.conf.default.accept_dad=0' >> /etc/sysctl.conf +echo 'net.ipv6.route.max_size=65536' >> /etc/sysctl.conf +echo 'net.ipv6.neigh.default.retrans_time_ms=1000' >> /etc/sysctl.conf +{{- end }} + +# ------------------------ 3. Tweaks for Specific Networking Configurations ----- + +#See https://help.aliyun.com/document_detail/118806.html#uicontrol-e50-ddj-w0y +sed -r -i "s@#{0,}?net.bridge.bridge-nf-call-arptables ?= ?(0|1)@net.bridge.bridge-nf-call-arptables = 1@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?vm.max_map_count ?= ?([0-9]{1,})@vm.max_map_count = 262144@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?vm.swappiness ?= ?([0-9]{1,})@vm.swappiness = 0@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?fs.inotify.max_user_instances ?= ?([0-9]{1,})@fs.inotify.max_user_instances = 524288@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?kernel.pid_max ?= ?([0-9]{1,})@kernel.pid_max = 65535@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?vm.overcommit_memory ?= ?(0|1|2)@vm.overcommit_memory = 0@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?fs.inotify.max_user_watches ?= ?([0-9]{1,})@fs.inotify.max_user_watches = 524288@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?fs.pipe-max-size ?= ?([0-9]{1,})@fs.pipe-max-size = 4194304@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.core.netdev_max_backlog ?= ?([0-9]{1,})@net.core.netdev_max_backlog = 65535@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.core.rmem_max ?= ?([0-9]{1,})@net.core.rmem_max = 33554432@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.core.wmem_max ?= ?([0-9]{1,})@net.core.wmem_max = 33554432@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.core.somaxconn ?= ?([0-9]{1,})@net.core.somaxconn = 32768@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?fs.aio-max-nr ?= ?([0-9]{1,})@fs.aio-max-nr = 262144@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?kernel.watchdog_thresh ?= ?([0-9]{1,})@kernel.watchdog_thresh = 5@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?kernel.hung_task_timeout_secs ?= ?([0-9]{1,})@kernel.hung_task_timeout_secs = 5@g" /etc/sysctl.conf +{{- if .internal_ipv4 | empty | not }} +sed -r -i "s@#{0,}?net.ipv4.tcp_tw_recycle ?= ?(0|1|2)@net.ipv4.tcp_tw_recycle = 0@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.tcp_tw_reuse ?= ?(0|1)@net.ipv4.tcp_tw_reuse = 0@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.conf.all.rp_filter ?= ?(0|1|2)@net.ipv4.conf.all.rp_filter = 1@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.conf.default.rp_filter ?= ?(0|1|2)@net.ipv4.conf.default.rp_filter = 1@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.ip_forward ?= ?(0|1)@net.ipv4.ip_forward = 1@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.bridge.bridge-nf-call-iptables ?= ?(0|1)@net.bridge.bridge-nf-call-iptables = 1@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.ip_local_reserved_ports ?= ?([0-9]{1,}-{0,1},{0,1}){1,}@net.ipv4.ip_local_reserved_ports = 30000-32767@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.tcp_max_syn_backlog ?= ?([0-9]{1,})@net.ipv4.tcp_max_syn_backlog = 1048576@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.neigh.default.gc_thresh1 ?= ?([0-9]{1,})@net.ipv4.neigh.default.gc_thresh1 = 512@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.neigh.default.gc_thresh2 ?= ?([0-9]{1,})@net.ipv4.neigh.default.gc_thresh2 = 2048@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.neigh.default.gc_thresh3 ?= ?([0-9]{1,})@net.ipv4.neigh.default.gc_thresh3 = 4096@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.conf.eth0.arp_accept ?= ?(0|1)@net.ipv4.conf.eth0.arp_accept = 1@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.tcp_retries2 ?= ?([0-9]{1,})@net.ipv4.tcp_retries2 = 15@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.tcp_max_tw_buckets ?= ?([0-9]{1,})@net.ipv4.tcp_max_tw_buckets = 1048576@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.tcp_max_orphans ?= ?([0-9]{1,})@net.ipv4.tcp_max_orphans = 65535@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.udp_rmem_min ?= ?([0-9]{1,})@net.ipv4.udp_rmem_min = 131072@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.udp_wmem_min ?= ?([0-9]{1,})@net.ipv4.udp_wmem_min = 131072@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.conf.all.arp_ignore ?= ??(0|1|2)@net.ipv4.conf.all.arp_ignore = 1@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.conf.default.arp_ignore ?= ??(0|1|2)@net.ipv4.conf.default.arp_ignore = 1@g" /etc/sysctl.conf +{{- end }} +{{- if .internal_ipv6 | empty | not }} +#add for ipv6 +sed -r -i "s@#{0,}?net.bridge.bridge-nf-call-ip6tables ?= ?(0|1)@net.bridge.bridge-nf-call-ip6tables = 1@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.conf.all.disable_ipv6 ?= ?([0-9]{1,})@net.ipv6.conf.all.disable_ipv6 = 0@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.conf.default.disable_ipv6 ?= ?([0-9]{1,})@net.ipv6.conf.default.disable_ipv6 = 0@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.conf.lo.disable_ipv6 ?= ?([0-9]{1,})@net.ipv6.conf.lo.disable_ipv6 = 0@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.conf.all.forwarding ?= ?([0-9]{1,})@net.ipv6.conf.all.forwarding = 1@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.conf.default.accept_dad ?= ?([0-9]{1,})@net.ipv6.conf.default.accept_dad = 0@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.route.max_size ?= ?([0-9]{1,})@net.ipv6.route.max_size = 65536@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.neigh.default.retrans_time_ms ?= ?([0-9]{1,})@net.ipv6.neigh.default.retrans_time_ms = 1000@g" /etc/sysctl.conf +{{- end }} + +tmpfile="$$.tmp" +awk ' !x[$0]++{print > "'$tmpfile'"}' /etc/sysctl.conf +mv $tmpfile /etc/sysctl.conf + +# ------------------------ 4. Security Limit ------------------------------------ + +# ulimit +echo "* soft nofile 1048576" >> /etc/security/limits.conf +echo "* hard nofile 1048576" >> /etc/security/limits.conf +echo "* soft nproc 65536" >> /etc/security/limits.conf +echo "* hard nproc 65536" >> /etc/security/limits.conf +echo "* soft memlock unlimited" >> /etc/security/limits.conf +echo "* hard memlock unlimited" >> /etc/security/limits.conf + +sed -r -i "s@#{0,}?\* soft nofile ?([0-9]{1,})@\* soft nofile 1048576@g" /etc/security/limits.conf +sed -r -i "s@#{0,}?\* hard nofile ?([0-9]{1,})@\* hard nofile 1048576@g" /etc/security/limits.conf +sed -r -i "s@#{0,}?\* soft nproc ?([0-9]{1,})@\* soft nproc 65536@g" /etc/security/limits.conf +sed -r -i "s@#{0,}?\* hard nproc ?([0-9]{1,})@\* hard nproc 65536@g" /etc/security/limits.conf +sed -r -i "s@#{0,}?\* soft memlock ?([0-9]{1,}([TGKM]B){0,1}|unlimited)@\* soft memlock unlimited@g" /etc/security/limits.conf +sed -r -i "s@#{0,}?\* hard memlock ?([0-9]{1,}([TGKM]B){0,1}|unlimited)@\* hard memlock unlimited@g" /etc/security/limits.conf + +tmpfile="$$.tmp" +awk ' !x[$0]++{print > "'$tmpfile'"}' /etc/security/limits.conf +mv $tmpfile /etc/security/limits.conf + +# ------------------------ 5. Firewall Configurations --------------------------- + +if systemctl is-active firewalld --quiet; then + systemctl stop firewalld 1>/dev/null 2>/dev/null + systemctl disable firewalld 1>/dev/null 2>/dev/null +fi +if systemctl is-active ufw --quiet; then + systemctl stop ufw 1>/dev/null 2>/dev/null + systemctl disable ufw 1>/dev/null 2>/dev/null +fi + +# ------------------------ 6. System Module Settings ---------------------------- + +modinfo br_netfilter > /dev/null 2>&1 +if [ $? -eq 0 ]; then + modprobe br_netfilter + mkdir -p /etc/modules-load.d + echo 'br_netfilter' > /etc/modules-load.d/kubekey-br_netfilter.conf +fi + +modinfo overlay > /dev/null 2>&1 +if [ $? -eq 0 ]; then + modprobe overlay + echo 'overlay' >> /etc/modules-load.d/kubekey-br_netfilter.conf +fi + +# ------------------------ 7. IPTables and Connection Tracking ----------------- + +modprobe ip_vs +modprobe ip_vs_rr +modprobe ip_vs_wrr +modprobe ip_vs_sh + +cat > /etc/modules-load.d/kube_proxy-ipvs.conf << EOF +ip_vs +ip_vs_rr +ip_vs_wrr +ip_vs_sh +EOF + +modprobe nf_conntrack_ipv4 1>/dev/null 2>/dev/null +if [ $? -eq 0 ]; then + echo 'nf_conntrack_ipv4' > /etc/modules-load.d/kube_proxy-ipvs.conf +else + modprobe nf_conntrack + echo 'nf_conntrack' > /etc/modules-load.d/kube_proxy-ipvs.conf +fi +sysctl -p + +# ------------------------ 8. Local Host DNS Configuration --------------------- + +sed -i ':a;$!{N;ba};s@# kubekey hosts BEGIN.*# kubekey hosts END@@' /etc/hosts +sed -i '/^$/N;/\n$/N;//D' /etc/hosts + +cat >>/etc/hosts< /proc/sys/vm/drop_caches + +# Make sure the iptables utility doesn't use the nftables backend. +{{- if .internal_ipv4 | empty | not }} +update-alternatives --set iptables /usr/sbin/iptables-legacy >/dev/null 2>&1 || true +{{- end }} +{{- if .internal_ipv6 | empty | not }} +update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy >/dev/null 2>&1 || true +{{- end }} +update-alternatives --set arptables /usr/sbin/arptables-legacy >/dev/null 2>&1 || true +update-alternatives --set ebtables /usr/sbin/ebtables-legacy >/dev/null 2>&1 || true diff --git a/builtin/capkk/roles/install/cloud-config/defaults/main.yaml b/builtin/capkk/roles/install/cloud-config/defaults/main.yaml new file mode 100644 index 000000000..14d625151 --- /dev/null +++ b/builtin/capkk/roles/install/cloud-config/defaults/main.yaml @@ -0,0 +1,8 @@ +kubernetes: + control_plane_endpoint: + kube_vip: + image: + registry: >- + {{ .dockerio_registry }} + repository: plndr/kube-vip + tag: v0.7.2 \ No newline at end of file diff --git a/builtin/capkk/roles/install/cloud-config/tasks/main.yaml b/builtin/capkk/roles/install/cloud-config/tasks/main.yaml new file mode 100644 index 000000000..e5394b276 --- /dev/null +++ b/builtin/capkk/roles/install/cloud-config/tasks/main.yaml @@ -0,0 +1,151 @@ +--- +- name: Install kube-vip + when: + - eq .kubernetes.control_plane_endpoint.type "kube_vip" + - or (.kubernetes.roles | has "master") (.kubernetes.roles | has "control-plane") + template: + src: kube-vip.yaml + dest: /etc/kubernetes/manifests/kube-vip.yaml + +# not support "encoding","append","owner". +- name: Deal cloud-config write_files + loop: "{{ .cloud_config.write_files | toJson }}" + copy: + content: | + {{ .item.content }} + dest: "{{ .item.path }}" + mode: "{{ .item.permissions }}" + +- name: Deal cloud-config users + loop: "{{ .cloud_config.users | toJson }}" + command: | + #!/bin/bash + + if id "{{ .item.name }}" &>/dev/null; then + echo "User '{{ .item.name }}' already exists" + exit 0 + fi + # Create user '{{ .item.name }}' with bash shell and home directory + useradd "{{ .item.name }}" + {{- if .item.passwd }} + # Set password + echo "{{ .item.name }}:{{ .item.passwd }}" | chpasswd + {{- end }} + {{- if .item.gecos }} + # Set gecos information + usermod -c "{{ .item.gecos }}" "{{ .item.name }}" + {{- end }} + {{- if .item.home_dir }} + # Set home directory + usermod -d "{{ .item.home_dir }}" "{{ .item.name }}" + {{- end }} + {{- if .item.shell }} + # Set shell + usermod -s "{{ .item.shell }}" "{{ .item.name }}" + {{- end }} + {{- if .item.primary_group }} + # Set primary group + usermod -g "{{ .item.primary_group }}" "{{ .item.name }}" + {{- end }} + {{- if .item.lock_passwd }} + # Lock password + usermod -L "{{ .item.name }}" + {{- end }} + {{- if .item.groups }} + # Add user to groups + usermod -aG "{{ .item.groups }}" "{{ .item.name }}" + {{- end }} + {{- if .item.sudo }} + # Add user to sudoers + echo "{{ .item.name }} {{ .item.sudo }}" > "/etc/sudoers.d/{{ .item.name }}" + {{- end }} + {{- if .item.ssh_authorized_keys }} + # Add SSH authorized keys + mkdir -p "{{ .item.home_dir }}/.ssh" + echo "{{ .item.ssh_authorized_keys }}" > "{{ .item.home_dir }}/.ssh/authorized_keys" + chown -R "{{ .item.name }}" "{{ .item.home_dir }}/.ssh" + chmod 700 "{{ .item.home_dir }}/.ssh" + chmod 600 "{{ .item.home_dir }}/.ssh/authorized_keys" + {{- end }} + +- name: Deal cloud-config disk_setup + when: .cloud_config.disk_setup + command: | + #!/bin/bash + + {{- range $_, $disk := .cloud_config.disk_setup }} + if lsblk | grep -q "{{ $disk.device }}"; then + echo "Disk {{ $disk.device }} already configured, skipping." + else + echo "Configuring disk {{ $disk.device }}" + # setup disk on '{{ $disk.device }}' + {{- if equal $disk.table_type "gpt" }} + parted "{{ $disk.device }}" mklabel gpt + {{- else equal $disk.table_type "mbr" }} + parted "{{ $disk.device }}" mklabel msdos + {{- end }} + {{- if $disk.layout }} + # create a single partition for the entire disk + parted -a optimal "{{ $disk.device }}" mkpart primary ext4 0% 100% + {{- end }} + fi + {{- end }} + +- name: Deal cloud-config fs_setup + loop: "{{ .cloud_config.fs_setup | toJson }}" + command: | + #!/bin/bash + + DEVICE="{{ .item.device }}" + {{- if .item.partition }} + DEVICE="${DEVICE}{{ .item.partition | atoi }}" + {{- end }} + + if blkid "$DEVICE" &>/dev/null; then + echo "Filesystem already exists on $DEVICE" + {{- if .item.overwrite }} + # Overwrite existing filesystem on '$DEVICE' + mkfs -t "{{ .item.filesystem }}" '$DEVICE' + {{- else }} + else + echo "Creating filesystem on $DEVICE..." + mkfs -t "{{ .item.filesystem }}" {{- if .item.label }}-L "{{ .item.label }}"{{ end }} {{- range .item.extra_opts }}"{{ . }}" {{ end }} "$DEVICE" + fi + +- name: Deal cloud-config mount + loop: "{{ .cloud_config.mounts | toJson }}" + command: | + #!/bin/bash + + MOUNT_POINT="{{ last .item }}" + if mountpoint -q "$MOUNT_POINT"; then + echo "Mount point $MOUNT_POINT already mounted, skipping." + else + echo "Mounting {{ first .item }} to $MOUNT_POINT..." + mount -L "{{ first .item }}" "$MOUNT_POINT" + echo "LABEL={{ first .item }} $MOUNT_POINT ext4 defaults 0 0" >> /etc/fstab + fi + +- name: Deal runcmd + loop: "{{ .cloud_config.runcmd | toJson }}" + command: "{{ .item }}" + +- name: Sync kubeconfig + copy: + src: >- + {{ .cloud_config_dir }}/kubeconfig/value + dest: /root/.kube/config + mode: 0600 + +- name: Label kubernetes role + loop: "{{ .kubernetes.roles | toJson }}" + command: | + #!/bin/bash + kubectl label node "{{ .hostname }}" node-role.kubernetes.io/{{ .item }}="" --overwrite + +- name: Remove traint if node is worker + when: .kubernetes.roles | has "worker" + ignore_errors: true + command: | + kubectl taint nodes "{{ .hostname }}" node-role.kubernetes.io/master- + kubectl taint nodes "{{ .hostname }}" node-role.kubernetes.io/control-plane- \ No newline at end of file diff --git a/builtin/capkk/roles/install/cloud-config/templates/kube-vip.yaml b/builtin/capkk/roles/install/cloud-config/templates/kube-vip.yaml new file mode 100644 index 000000000..45752e679 --- /dev/null +++ b/builtin/capkk/roles/install/cloud-config/templates/kube-vip.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +kind: Pod +metadata: + name: kube-vip + namespace: kube-system +spec: + containers: + - args: + - manager + env: + - name: address + value: {{ .kubernetes.control_plane_endpoint.kube_vip.address }} + - name: vip_interface + value: "" + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "true" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: lb_enable + value: "true" + - name: lb_port + value: "6443" + image: {{ .kubernetes.control_plane_endpoint.kube_vip.image.registry }}/{{ .kubernetes.control_plane_endpoint.kube_vip.image.repository }}:{{ .kubernetes.control_plane_endpoint.kube_vip.image.tag }} + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostNetwork: true + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + type: FileOrCreate + name: kubeconfig \ No newline at end of file diff --git a/builtin/capkk/roles/install/cni/defaults/main.yaml b/builtin/capkk/roles/install/cni/defaults/main.yaml new file mode 100644 index 000000000..00ff1d157 --- /dev/null +++ b/builtin/capkk/roles/install/cni/defaults/main.yaml @@ -0,0 +1,113 @@ +cni: + type: >- + {{ .kubernetes.kube_network_plugin | default "calico" }} + # ip cidr config. + # In Kubernetes, the Pod CIDR supports both IPv4 and IPv6 configurations. It can be specified as follows: + # "Single-stack IPv4": the pod_cidr value format "ipv4" + # "Single-stack IPv6": the pod_cidr value format "ipv6" + # "Dual-stack (IPv4 and IPv6)": the pod_cidr value format "ipv4,ipv6" + ipv4_support: >- + {{ eq (.cluster_network.pods.cidrBlocks | first | ipFamily) "IPv4" }} + ipv4_pods_cidr: >- + {{- if eq (.cluster_network.pods.cidrBlocks | first | ipFamily) "IPv4" -}} + {{ .cluster_network.pods.cidrBlocks | first }} + {{- end -}} + ipv4_block_size: 24 + ipv6_support: >- + {{ eq (.cluster_network.pods.cidrBlocks | last | ipFamily) "IPv6" }} + ipv6_pods_cidr: >- + {{- if eq (.cluster_network.pods.cidrBlocks | last | ipFamily) "IPv6" -}} + {{ .cluster_network.pods.cidrBlocks | last }} + {{- end -}} + ipv6_block_size: 120 + kube_svc_cidr: >- + {{ .cluster_network.service.cidrBlocks | join "," }} + calico: + values: | + # calico helm values + installation: + registry: {{ .dockerio_registry }} + calicoNetwork: + bgp: Enabled + cilium: + values: | + # cilium helm values + image: + repository: {{ .quayio_registry }}/cilium/cilium-cli + certgen: + image: + repository: {{ .quayio_registry }}/cilium/certgen + hubble: + relay: + image: + repository: {{ .quayio_registry }}/cilium/hubble-relay-ci + ui: + backend: + image: + repository: {{ .quayio_registry }}/cilium/hubble-ui-backend + frontend: + image: + repository: {{ .quayio_registry }}/cilium/hubble-ui + envoy: + image: + repository: {{ .quayio_registry }}/cilium/cilium-envoy + operator: + replicas: 2 + image: + repository: {{ .quayio_registry }}/cilium/operator + nodeinit: + image: + repository: {{ .quayio_registry }}/cilium/startup-script + preflight: + image: + repository: {{ .quayio_registry }}/cilium/cilium-ci + clustermesh: + apiserver: + image: + repository: {{ .quayio_registry }}/cilium/clustermesh-apiserver-ci + authentication: + mutual: + spire: + install: + initImage: + repository: {{ .dockerio_registry }}/library/busybox + agent: + image: + repository: {{ .ghcrio_registry }}/spiffe/spire-agent + server: + image: + repository: {{ .ghcrio_registry }}/spiffe/spire-server + ipv4: + enabled: {{ .cni.ipv4_support }} + ipv6: + enabled: {{ .cni.ipv6_support }} + ipam: + operator: + {{- if .cni.ipv4_support }} + clusterPoolIPv4PodCIDRList: + - {{ .cni.ipv4_pods_cidr }} + clusterPoolIPv4MaskSize: {{ .cni.ipv4_block_size }} + {{- end }} + {{- if .cni.ipv6_support }} + clusterPoolIPv6PodCIDRList: + - {{ .cni.ipv6_pods_cidr }} + clusterPoolIPv6MaskSize: {{ .cni.ipv6_block_size }} + {{- end }} + {{- if not (.kubernetes.kube_proxy.enabled | default true) }} + kubeProxyReplacement: "true" + k8sServiceHost: {{ .kubernetes.control_plane_endpoint.host }} + k8sServicePort: {{ .kubernetes.control_plane_endpoint.port }} + {{- end }} + flannel: + # https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md + values: | + # flannel helm values + podCidr: {{ .cni.ipv4_pod_cidr }} + podCidrv6: {{ .cni.ipv6_pod_cidr }} + flannel: + image: + repository: {{ .dockerio_registry }}/flannel/flannel + image_cni: + repository: {{ .dockerio_registry }}/flannel/flannel-cni-plugin + # support "vxlan" and "host-gw" + backend: vxlan \ No newline at end of file diff --git a/builtin/capkk/roles/install/cni/tasks/calico.yaml b/builtin/capkk/roles/install/cni/tasks/calico.yaml new file mode 100644 index 000000000..82bfb0585 --- /dev/null +++ b/builtin/capkk/roles/install/cni/tasks/calico.yaml @@ -0,0 +1,32 @@ +--- +- name: Check if calicoctl is installed + ignore_errors: true + command: calicoctl version + register: calicoctl_install_version + register_type: yaml +- name: Install calicoctl + when: .calicoctl_install_version.error | empty | not + block: + - name: Sync calicoctl to remote + copy: + src: >- + {{ .binary_dir }}/cni/calico/{{ .calico_version }}/{{ .binary_type }}/calicoctl + dest: /usr/local/bin/calicoctl + mode: 0755 + +- name: Sync calico package to remote + copy: + src: >- + {{ .binary_dir }}/cni/calico/tigera-operator-{{ .calico_version }}.tgz + dest: >- + /etc/kubernetes/cni/tigera-operator-{{ .calico_version }}.tgz + +- name: Generate calico custom value file + copy: + content: | + {{ .cni.calico.values }} + dest: /etc/kubernetes/cni/calico-values.yaml + +- name: Apply calico + command: | + helm upgrade --install --create-namespace --namespace tigera-operator calico /etc/kubernetes/cni/tigera-operator-{{ .calico_version }}.tgz -f /etc/kubernetes/cni/calico-values.yaml diff --git a/builtin/capkk/roles/install/cni/tasks/cilium.yaml b/builtin/capkk/roles/install/cni/tasks/cilium.yaml new file mode 100644 index 000000000..31bdb4eaa --- /dev/null +++ b/builtin/capkk/roles/install/cni/tasks/cilium.yaml @@ -0,0 +1,25 @@ +--- +- name: Sync cilium cli package + when: .ciliumcli_version | empty | not + copy: + src: >- + {{ .binary_dir }}/cni/cilium/ciliumcli-{{ .ciliumcli_version }}/{{ .item }} + dest: /usr/local/bin/cilium + +- name: Sync cilium helm chart package + copy: + src: >- + {{ .binary_dir }}/cni/cilium/cilium-{{ .cilium_version }}.tgz + dest: >- + /etc/kubernetes/cni/cilium-{{ .cilium_version }}.tgz + +- name: Sync cilium helm chart custom value file + copy: + content: | + {{ .cni.cilium.values }} + dest: /etc/kubernetes/cni/cilium-values.yaml + +# https://docs.cilium.io/en/stable/installation/k8s-install-helm/ +- name: Install cilium + command: | + helm upgrade --install --namespace kube-system cilium /etc/kubernetes/cni/cilium-{{ .cilium_version }}.tgz -f /etc/kubernetes/cni/cilium-values.yaml diff --git a/builtin/capkk/roles/install/cni/tasks/flannel.yaml b/builtin/capkk/roles/install/cni/tasks/flannel.yaml new file mode 100644 index 000000000..9e11e75e9 --- /dev/null +++ b/builtin/capkk/roles/install/cni/tasks/flannel.yaml @@ -0,0 +1,17 @@ +--- +# https://github.com/flannel-io/flannel/blob/master/Documentation/kubernetes.md +- name: Sync flannel package to remote + copy: + src: >- + {{ .binary_dir }}/cni/flannel/flannel.tgz + dest: /etc/kubernetes/cni/flannel.tgz + +- name: Generate flannel custom value file + copy: + content: | + {{ .cni.flannel.values }} + dest: /etc/kubernetes/cni/flannel-values.yaml + +- name: Apply flannel + command: | + helm upgrade --install --create-namespace --namespace kube-flannel flannel /etc/kubernetes/cni/flannel.tgz -f /etc/kubernetes/cni/flannel-values.yaml diff --git a/builtin/capkk/roles/install/cni/tasks/main.yaml b/builtin/capkk/roles/install/cni/tasks/main.yaml new file mode 100644 index 000000000..140766da6 --- /dev/null +++ b/builtin/capkk/roles/install/cni/tasks/main.yaml @@ -0,0 +1,20 @@ +--- +- name: check cni by helm + command: helm list -a -A -q -o json 2>/dev/null + register: installed_helm_packages + register_type: json + +- include_tasks: calico.yaml + when: + - .cni.type | eq "calico" + - .installed_helm_packages.stdout | has "calico" | not + +- include_tasks: flannel.yaml + when: + - .cni.type | eq "flannel" + - .installed_helm_packages.stdout | has "flannel" | not + +- include_tasks: cilium.yaml + when: + - .cni.type | eq "cilium" + - .installed_helm_packages.stdout | has "cilium" | not diff --git a/builtin/capkk/roles/install/cri/defaults/main.yaml b/builtin/capkk/roles/install/cri/defaults/main.yaml new file mode 100644 index 000000000..a948a2ca2 --- /dev/null +++ b/builtin/capkk/roles/install/cri/defaults/main.yaml @@ -0,0 +1,24 @@ +cri: + # support: systemd, cgroupfs + cgroup_driver: systemd + sandbox_image: + registry: >- + {{ .dockerio_registry }} + repository: kubesphere/pause + tag: 3.5 + # support: containerd,docker,crio + # the endpoint of containerd + cri_socket: >- + {{- if .cri.container_manager | eq "containerd" -}} + unix:///var/run/containerd/containerd.sock + {{- else if and (.cri.container_manager | eq "docker") (.kube_version | semverCompare ">=v1.24.0") -}} + unix:///var/run/cri-dockerd.sock + {{- end -}} + containerd: + data_root: /var/lib/containerd + docker: + data_root: /var/lib/docker + registry: + mirrors: ["https://registry-1.docker.io"] + insecure_registries: [] + auths: [] \ No newline at end of file diff --git a/builtin/capkk/roles/install/cri/files/containerd.service b/builtin/capkk/roles/install/cri/files/containerd.service new file mode 100644 index 000000000..5f67110ab --- /dev/null +++ b/builtin/capkk/roles/install/cri/files/containerd.service @@ -0,0 +1,26 @@ +[Unit] +Description=containerd container runtime +Documentation=https://containerd.io +After=network.target local-fs.target + +[Service] +ExecStartPre=-/sbin/modprobe overlay +ExecStart=/usr/local/bin/containerd + +Type=notify +Delegate=yes +KillMode=process +Restart=always +RestartSec=5 +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNPROC=infinity +LimitCORE=infinity +LimitNOFILE=1048576 +# Comment TasksMax if your systemd version does not supports it. +# Only systemd 226 and above support this version. +TasksMax=infinity +OOMScoreAdjust=-999 + +[Install] +WantedBy=multi-user.target diff --git a/builtin/capkk/roles/install/cri/files/docker.service b/builtin/capkk/roles/install/cri/files/docker.service new file mode 100644 index 000000000..929b8a436 --- /dev/null +++ b/builtin/capkk/roles/install/cri/files/docker.service @@ -0,0 +1,47 @@ +[Unit] +Description=Docker Application Container Engine +Documentation=https://docs.docker.com +# After=network-online.target firewalld.service containerd.service +# Wants=network-online.target +# Requires=docker.socket containerd.service + +[Service] +Type=notify +# the default is not to use systemd for cgroups because the delegate issues still +# exists and systemd currently does not support the cgroup feature set required +# for containers run by docker +ExecStart=/usr/local/bin/dockerd --containerd=/run/containerd/containerd.sock +ExecReload=/bin/kill -s HUP $MAINPID +TimeoutSec=0 +RestartSec=2 +Restart=always + +# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229. +# Both the old, and new location are accepted by systemd 229 and up, so using the old location +# to make them work for either version of systemd. +StartLimitBurst=3 + +# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230. +# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make +# this option work for either version of systemd. +StartLimitInterval=60s + +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity + +# Comment TasksMax if your systemd version does not support it. +# Only systemd 226 and above support this option. +TasksMax=infinity + +# set delegate yes so that systemd does not reset the cgroups of docker containers +Delegate=yes + +# kill only the docker process, not all processes in the cgroup +KillMode=process +OOMScoreAdjust=-500 + +[Install] +WantedBy=multi-user.target diff --git a/builtin/capkk/roles/install/cri/tasks/install_containerd.yaml b/builtin/capkk/roles/install/cri/tasks/install_containerd.yaml new file mode 100644 index 000000000..6f822f1f4 --- /dev/null +++ b/builtin/capkk/roles/install/cri/tasks/install_containerd.yaml @@ -0,0 +1,40 @@ +--- +- name: Check if runc is installed + ignore_errors: true + command: runc --version + register: runc_install_version +- name: Sync runc binary to remote + when: or (.runc_install_version.error | empty | not) (.runc_install_version.stdout | contains (printf "runc version %s\n" (.runc_version | default "" | trimPrefix "v" )) | not) + copy: + src: >- + {{ .binary_dir }}/runc/{{ .runc_version }}/{{ .binary_type }}/runc.{{ .binary_type }} + dest: /usr/local/bin/runc + mode: 0755 + +- name: Check if containerd is installed + ignore_errors: true + command: containerd --version + register: containerd_install_version +- name: Install containerd + when: or (.containerd_install_version.error | empty | not) (.containerd_install_version.stdout | contains (printf " %s " .containerd_version) | not) + block: + - name: Sync containerd binary to remote + copy: + src: >- + {{ .binary_dir }}/containerd/{{ .containerd_version }}/{{ .binary_type }}/containerd-{{ .containerd_version | default "" | trimPrefix "v" }}-linux-{{ .binary_type }}.tar.gz + dest: >- + {{ .tmp_dir }}/containerd-{{ .containerd_version | default "" | trimPrefix "v" }}-linux-{{ .binary_type }}.tar.gz + - name: Unpackage containerd binary + command: | + tar -xvf {{ .tmp_dir }}/containerd-{{ .containerd_version | default "" | trimPrefix "v" }}-linux-{{ .binary_type }}.tar.gz --strip-components=1 -C /usr/local/bin/ + - name: Generate containerd config file + template: + src: containerd.config + dest: /etc/containerd/config.toml + - name: Generate containerd Service file + copy: + src: containerd.service + dest: /etc/systemd/system/containerd.service + - name: Start containerd + command: | + systemctl daemon-reload && systemctl start containerd.service && systemctl enable containerd.service \ No newline at end of file diff --git a/builtin/capkk/roles/install/cri/tasks/install_crictl.yaml b/builtin/capkk/roles/install/cri/tasks/install_crictl.yaml new file mode 100644 index 000000000..36b6f27ae --- /dev/null +++ b/builtin/capkk/roles/install/cri/tasks/install_crictl.yaml @@ -0,0 +1,22 @@ +--- +- name: Check if crictl is installed + ignore_errors: true + command: crictl --version + register: crictl_install_version + +- name: Install crictl + when: or (.crictl_install_version.error | empty | not) (.crictl_install_version.stdout | ne (printf "crictl version %s" .crictl_version)) + block: + - name: Sync crictl binary to remote + copy: + src: >- + {{ .binary_dir }}/crictl/{{ .crictl_version }}/{{ .binary_type }}/crictl-{{ .crictl_version }}-linux-{{ .binary_type }}.tar.gz + dest: >- + {{ .tmp_dir }}/crictl-{{ .crictl_version }}-linux-{{ .binary_type }}.tar.gz + - name: Unpackage crictl binary + command: | + tar -xvf {{ .tmp_dir }}/crictl-{{ .crictl_version }}-linux-{{ .binary_type }}.tar.gz -C /usr/local/bin/ + - name: Generate crictl config file + template: + src: crictl.config + dest: /etc/crictl.yaml diff --git a/builtin/capkk/roles/install/cri/tasks/install_cridockerd.yaml b/builtin/capkk/roles/install/cri/tasks/install_cridockerd.yaml new file mode 100644 index 000000000..b50777aa9 --- /dev/null +++ b/builtin/capkk/roles/install/cri/tasks/install_cridockerd.yaml @@ -0,0 +1,25 @@ +--- +- name: Check if cri-dockerd is installed + ignore_errors: true + command: cri-dockerd --version + register: cridockerd_install_version + +- name: Install cri-dockerd + when: or (.cridockerd_install_version.error | empty | not) (.cridockerd_install_version.stdout | hasPrefix (printf "cri-dockerd %s " .cridockerd_version) | not) + block: + - name: Sync cri-dockerd Binary to remote + copy: + src: >- + {{ .binary_dir }}/cri-dockerd/{{ .cridockerd_version }}/{{ .binary_type }}/cri-dockerd-{{ .cridockerd_version | default "" | trimPrefix "v" }}.{{ .binary_type }}.tgz + dest: >- + {{ .tmp_dir }}/cri-dockerd-{{ .cridockerd_version | default "" | trimPrefix "v" }}.{{ .binary_type }}.tgz + - name: Unpackage cri-dockerd binary + command: | + tar -xvf {{ .tmp_dir }}/cri-dockerd-{{ .cridockerd_version | default "" | trimPrefix "v" }}.{{ .binary_type }}.tgz --strip-components=1 -C /usr/local/bin/ + - name: Generate cri-dockerd Service file + template: + src: cri-dockerd.service + dest: /etc/systemd/system/cri-dockerd.service + - name: Start cri-dockerd service + command: | + systemctl daemon-reload && systemctl start cri-dockerd.service && systemctl enable cri-dockerd.service diff --git a/builtin/capkk/roles/install/cri/tasks/install_docker.yaml b/builtin/capkk/roles/install/cri/tasks/install_docker.yaml new file mode 100644 index 000000000..b1907c90a --- /dev/null +++ b/builtin/capkk/roles/install/cri/tasks/install_docker.yaml @@ -0,0 +1,34 @@ +--- +- name: Check if docker is installed + ignore_errors: true + command: docker --version + register: docker_install_version + +- name: Install docker + when: or (.docker_install_version.error | empty | not) (.docker_install_version.stdout | hasPrefix (printf "Docker version %s," .docker_version) | not) + block: + - name: Sync docker binary to remote + copy: + src: >- + {{ .binary_dir }}/docker/{{ .docker_version }}/{{ .binary_type }}/docker-{{ .docker_version }}.tgz + dest: >- + {{ .tmp_dir }}/docker-{{ .docker_version }}.tgz + - name: Unpackage docker binary + command: | + tar -C /usr/local/bin/ --strip-components=1 -xvf {{ .tmp_dir }}/docker-{{ .docker_version }}.tgz --wildcards docker/* + - name: Generate docker config file + template: + src: docker.config + dest: /etc/docker/daemon.json + - name: Generate docker service file + copy: + src: docker.service + dest: /etc/systemd/system/docker.service + - name: Generate containerd service file + copy: + src: containerd.service + dest: /etc/systemd/system/containerd.service + - name: Start docker service + command: | + systemctl daemon-reload && systemctl start containerd.service && systemctl enable containerd.service + systemctl daemon-reload && systemctl start docker.service && systemctl enable docker.service \ No newline at end of file diff --git a/builtin/capkk/roles/install/cri/tasks/main.yaml b/builtin/capkk/roles/install/cri/tasks/main.yaml new file mode 100644 index 000000000..956f4c2be --- /dev/null +++ b/builtin/capkk/roles/install/cri/tasks/main.yaml @@ -0,0 +1,17 @@ +--- +# install crictl +- include_tasks: install_crictl.yaml + +# install docker +- include_tasks: install_docker.yaml + when: .cri.container_manager | eq "docker" + +# install containerd +- include_tasks: install_containerd.yaml + when: .cri.container_manager | eq "containerd" + +# install cridockerd +- include_tasks: install_cridockerd.yaml + when: + - .cri.container_manager | eq "docker" + - .kube_version | semverCompare ">=v1.24.0" diff --git a/builtin/capkk/roles/install/cri/templates/containerd.config b/builtin/capkk/roles/install/cri/templates/containerd.config new file mode 100644 index 000000000..60da9ad14 --- /dev/null +++ b/builtin/capkk/roles/install/cri/templates/containerd.config @@ -0,0 +1,77 @@ +version = 2 + +root = "{{ .cri.containerd.data_root | default "/var/lib/containerd" }}" +state = "/run/containerd" + +[grpc] + address = "/run/containerd/containerd.sock" + uid = 0 + gid = 0 + max_recv_message_size = 16777216 + max_send_message_size = 16777216 + +[ttrpc] + address = "" + uid = 0 + gid = 0 + +[debug] + address = "" + uid = 0 + gid = 0 + level = "" + +[metrics] + address = "" + grpc_histogram = false + +[cgroup] + path = "" + +[timeouts] + "io.containerd.timeout.shim.cleanup" = "5s" + "io.containerd.timeout.shim.load" = "5s" + "io.containerd.timeout.shim.shutdown" = "3s" + "io.containerd.timeout.task.state" = "2s" + +[plugins] + [plugins."io.containerd.grpc.v1.cri"] + sandbox_image = "{{ .cri.sandbox_image.registry }}/{{ .cri.sandbox_image.repository }}:{{ .cri.sandbox_image.tag }}" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = {{ if .cri.cgroup_driver | eq "systemd" }}true{{ else }}false{{ end }} + [plugins."io.containerd.grpc.v1.cri".cni] + bin_dir = "/opt/cni/bin" + conf_dir = "/etc/cni/net.d" + max_conf_num = 1 + conf_template = "" + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] +{{- if .cri.registry.mirrors | empty | not }} + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = {{ .cri.registry.mirrors | toJson }} +{{- end }} +{{- range .cri.registry.insecure_registries }} + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ . }}"] + endpoint = ["http://{{ . }}"] +{{- end }} +{{- if .cri.registry.auths | empty | not }} + [plugins."io.containerd.grpc.v1.cri".registry.configs] + {{- range .cri.registry.auths }} + [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .repo }}".auth] + username = "{{ .username }}" + password = "{{ .password }}" + [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .repo }}".tls] + {{- if.ca_file }} + ca_file = {{ .ca_file }} + {{- end }} + {{- if .crt_file }} + cert_file = {{ .crt_file }} + {{- end }} + {{- if .key_file }} + key_file = {{ .key_file }} + {{- end }} + insecure_skip_verify = {{ .skip_ssl | default true }} + {{- end }} +{{- end }} diff --git a/builtin/capkk/roles/install/cri/templates/cri-dockerd.service b/builtin/capkk/roles/install/cri/templates/cri-dockerd.service new file mode 100644 index 000000000..244ef9a68 --- /dev/null +++ b/builtin/capkk/roles/install/cri/templates/cri-dockerd.service @@ -0,0 +1,36 @@ +[Unit] +Description=CRI Interface for Docker Application Container Engine +Documentation=https://docs.mirantis.com + +[Service] +Type=notify +ExecStart=/usr/local/bin/cri-dockerd --pod-infra-container-image "{{ .cri.sandbox_image.registry }}/{{ .cri.sandbox_image.repository }}:{{ .cri.sandbox_image.tag }}" +ExecReload=/bin/kill -s HUP $MAINPID +TimeoutSec=0 +RestartSec=2 +Restart=always + +# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229. +# Both the old, and new location are accepted by systemd 229 and up, so using the old location +# to make them work for either version of systemd. +StartLimitBurst=3 + +# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230. +# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make +# this option work for either version of systemd. +StartLimitInterval=60s + +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity + +# Comment TasksMax if your systemd version does not support it. +# Only systemd 226 and above support this option. +TasksMax=infinity +Delegate=yes +KillMode=process + +[Install] +WantedBy=multi-user.target diff --git a/builtin/capkk/roles/install/cri/templates/crictl.config b/builtin/capkk/roles/install/cri/templates/crictl.config new file mode 100644 index 000000000..9a8544e69 --- /dev/null +++ b/builtin/capkk/roles/install/cri/templates/crictl.config @@ -0,0 +1,5 @@ +runtime-endpoint: {{ .cri.cri_socket }} +image-endpoint: {{ .cri.cri_socket }} +timeout: 5 +debug: false +pull-image-on-create: false diff --git a/builtin/capkk/roles/install/cri/templates/docker.config b/builtin/capkk/roles/install/cri/templates/docker.config new file mode 100644 index 000000000..7eb1cc766 --- /dev/null +++ b/builtin/capkk/roles/install/cri/templates/docker.config @@ -0,0 +1,19 @@ +{ + "log-opts": { + "max-size": "5m", + "max-file":"3" + }, +{{- if .cri.docker.data_root }} + "data-root": "{{ .cri.docker.data_root }}", +{{- end }} +{{- if .cri.registry.mirrors }} + "registry-mirrors": {{ .cri.registry.mirrors | toJson }}, +{{- end }} + {{- if .cri.registry.insecure_registries }} + "insecure-registries": {{ .cri.registry.insecure_registries | toJson }}, +{{- end }} + {{- if .cri.docker.bridge_ip }} + "bip": "{{ .cri.docker.bridge_ip }}", +{{- end }} + "exec-opts": ["native.cgroupdriver={{ .cri.cgroup_driver }}"] +} diff --git a/builtin/capkk/roles/install/kubernetes/files/kubelet.service b/builtin/capkk/roles/install/kubernetes/files/kubelet.service new file mode 100644 index 000000000..ca57d677e --- /dev/null +++ b/builtin/capkk/roles/install/kubernetes/files/kubelet.service @@ -0,0 +1,15 @@ +[Unit] +Description=kubelet: The Kubernetes Node Agent +Documentation=http://kubernetes.io/docs/ + +[Service] +CPUAccounting=true +MemoryAccounting=true +ExecStart=/usr/local/bin/kubelet +Restart=always +StartLimitInterval=0 +RestartSec=10 + +[Install] +WantedBy=multi-user.target + diff --git a/builtin/capkk/roles/install/kubernetes/tasks/main.yaml b/builtin/capkk/roles/install/kubernetes/tasks/main.yaml new file mode 100644 index 000000000..a7a2e1e59 --- /dev/null +++ b/builtin/capkk/roles/install/kubernetes/tasks/main.yaml @@ -0,0 +1,80 @@ +--- +- name: Check if helm is installed + ignore_errors: true + command: helm version --template "{{ .Version }}" + register: helm_install_version +- name: Install helm + when: or (.helm_install_version.error | empty | not) (.helm_install_version.stdout | ne .helm_version) + block: + - name: Sync helm to remote + copy: + src: >- + {{ .binary_dir }}/helm/{{ .helm_version }}/{{ .binary_type }}/helm-{{ .helm_version }}-linux-{{ .binary_type }}.tar.gz + dest: >- + {{ .tmp_dir }}/helm-{{ .helm_version }}-linux-{{ .binary_type }}.tar.gz + - name: Install helm + command: | + tar --strip-components=1 -zxvf {{ .tmp_dir }}/helm-{{ .helm_version }}-linux-{{ .binary_type }}.tar.gz -C /usr/local/bin linux-{{ .binary_type }}/helm + +- name: Check if kubeadm is installed + ignore_errors: true + command: kubeadm version -o short + register: kubeadm_install_version +- name: Install kubeadm + when: or (.kubeadm_install_version.error | empty | not) (.kubeadm_install_version.stdout | ne .kube_version) + copy: + src: >- + {{ .binary_dir }}/kube/{{ .kube_version }}/{{ .binary_type }}/kubeadm + dest: /usr/local/bin/kubeadm + mode: 0755 + +- name: Check if kubectl is installed + ignore_errors: true + command: kubectl version --short + register: kubectl_install_version + register_type: yaml +- name: Sync kubectl to remote + when: | + or (.kubectl_install_version.error | empty | not) ((get .kubectl_install_version.stdout "Server Version") | ne .kube_version) + copy: + src: >- + {{ .binary_dir }}/kube/{{ .kube_version }}/{{ .binary_type }}/kubectl + dest: /usr/local/bin/kubectl + mode: 0755 + +- name: Check if kubelet is installed + ignore_errors: true + command: kubelet --version + register: kubelet_install_version +- name: Install kubelet + when: or (.kubelet_install_version.error | empty | not) (.kubelet_install_version.stdout | ne (printf "Kubernetes %s" .kube_version)) + block: + - name: Sync kubelet to remote + copy: + src: >- + {{ .binary_dir }}/kube/{{ .kube_version }}/{{ .binary_type }}/kubelet + dest: /usr/local/bin/kubelet + mode: 0755 + - name: Sync kubelet env to remote + template: + src: kubeadm/kubelet.env + dest: /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + - name: Sync kubelet service to remote + copy: + src: kubelet.service + dest: /etc/systemd/system/kubelet.service + - name: Register kubelet service + command: systemctl daemon-reload && systemctl enable kubelet.service + +- name: Install cni plugins + when: .cni_plugins_version | empty | not + block: + - name: Sync cni-plugin to remote + copy: + src: >- + {{ .binary_dir }}/cni/plugins/{{ .cni_plugins_version }}/{{ .binary_type }}/cni-plugins-linux-{{ .binary_type }}-{{ .cni_plugins_version }}.tgz + dest: >- + {{ .tmp_dir }}/cni-plugins-linux-{{ .binary_type }}-{{ .cni_plugins_version }}.tgz + - name: Install cni-plugin + command: | + tar -zxvf {{ .tmp_dir }}/cni-plugins-linux-{{ .binary_type }}-{{ .cni_plugins_version }}.tgz -C /opt/cni/bin/ \ No newline at end of file diff --git a/builtin/capkk/roles/install/kubernetes/templates/kubeadm/kubelet.env b/builtin/capkk/roles/install/kubernetes/templates/kubeadm/kubelet.env new file mode 100644 index 000000000..e8527cbf9 --- /dev/null +++ b/builtin/capkk/roles/install/kubernetes/templates/kubeadm/kubelet.env @@ -0,0 +1,18 @@ +# Note: This dropin only works with kubeadm and kubelet v1.11+ +[Service] +Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" +Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" +# This is a file that "kubeadm init" and "kubeadm join" generate at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically +EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env +# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use +# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file. +EnvironmentFile=-/etc/default/kubelet +{{- $internalIPv4 := .internal_ipv4 | default "" }} +{{- $internalIPv6 := .internal_ipv6 | default "" }} +{{- if $internalIPv4 | empty | not }} +Environment="KUBELET_EXTRA_ARGS=--node-ip={{ $internalIPv4 }} --hostname-override={{ .hostname }} {{ range $k,$v := .kubernetes.kubelet.extra_args }}--{{ $k }} {{ $v }} {{ end }}" +{{- else if $internalIPv6 | empty | not }} +Environment="KUBELET_EXTRA_ARGS=--node-ip={{ $internalIPv6 }} --hostname-override={{ .hostname }} {{ range $k,$v := .kubernetes.kubelet.extra_args }}--{{ $k }} {{ $v }} {{ end }}" +{{- end }} +ExecStart= +ExecStart=/usr/local/bin/kubelet --provider-id=kk://{{ .kubernetes.cluster_name }}/{{ .node_name }} $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS diff --git a/builtin/capkk/roles/install/storageclass/defaults/main.yaml b/builtin/capkk/roles/install/storageclass/defaults/main.yaml new file mode 100644 index 000000000..914fff1f5 --- /dev/null +++ b/builtin/capkk/roles/install/storageclass/defaults/main.yaml @@ -0,0 +1,21 @@ +sc: + local: + enabled: true + default: true + provisioner_image: + registry: >- + {{ .dockerio_registry }} + repository: openebs/provisioner-localpv + tag: 3.3.0 + linux_utils_image: + registry: >- + {{ .dockerio_registry }} + repository: openebs/linux-utils + tag: 3.3.0 + path: /var/openebs/local + nfs: # each k8s_cluster node should install nfs-utils + enabled: false + default: false + server: >- + {{ .groups.nfs | default list | first }} + path: /share/kubernetes diff --git a/builtin/capkk/roles/install/storageclass/tasks/local.yaml b/builtin/capkk/roles/install/storageclass/tasks/local.yaml new file mode 100644 index 000000000..4b09e706d --- /dev/null +++ b/builtin/capkk/roles/install/storageclass/tasks/local.yaml @@ -0,0 +1,9 @@ +--- +- name: Generate local manifest + template: + src: local-volume.yaml + dest: /etc/kubernetes/addons/local-volume.yaml + +- name: deploy local + command: | + kubectl apply -f /etc/kubernetes/addons/local-volume.yaml diff --git a/builtin/capkk/roles/install/storageclass/tasks/main.yaml b/builtin/capkk/roles/install/storageclass/tasks/main.yaml new file mode 100644 index 000000000..59bd771d4 --- /dev/null +++ b/builtin/capkk/roles/install/storageclass/tasks/main.yaml @@ -0,0 +1,6 @@ +--- +- include_tasks: local.yaml + when: .sc.local.enabled + +- include_tasks: nfs.yaml + when: .sc.nfs.enabled diff --git a/builtin/capkk/roles/install/storageclass/tasks/nfs.yaml b/builtin/capkk/roles/install/storageclass/tasks/nfs.yaml new file mode 100644 index 000000000..aae3288fd --- /dev/null +++ b/builtin/capkk/roles/install/storageclass/tasks/nfs.yaml @@ -0,0 +1,13 @@ +--- +- name: Sync nfs provisioner helm to remote + copy: + src: >- + {{ .work_dir }}/kubekey/sc/nfs-subdir-external-provisioner-{{ .nfs_provisioner_version }}.tgz + dest: >- + /etc/kubernetes/addons/nfs-subdir-external-provisioner-{{ .nfs_provisioner_version }}.tgz + +- name: Deploy nfs provisioner + command: | + helm upgrade --install nfs-subdir-external-provisioner /etc/kubernetes/addons/nfs-subdir-external-provisioner-{{ .nfs_provisioner_version }}.tgz --namespace kube-system \ + --set nfs.server={{ .sc.nfs.server }} --set nfs.path={{ .sc.nfs.path }} \ + --set storageClass.defaultClass={{ if .sc.local.default }}true{{ else }}false{{ end }} diff --git a/builtin/capkk/roles/install/storageclass/templates/local-volume.yaml b/builtin/capkk/roles/install/storageclass/templates/local-volume.yaml new file mode 100644 index 000000000..717bf8fe7 --- /dev/null +++ b/builtin/capkk/roles/install/storageclass/templates/local-volume.yaml @@ -0,0 +1,153 @@ +--- +#Sample storage classes for OpenEBS Local PV +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: local + annotations: + storageclass.kubesphere.io/supported-access-modes: '["ReadWriteOnce"]' + storageclass.beta.kubernetes.io/is-default-class: "{{ if .sc.local.default }}true{{ else }}false{{ end }}" + openebs.io/cas-type: local + cas.openebs.io/config: | + - name: StorageType + value: "hostpath" + - name: BasePath + value: "{{ .sc.local.path }}" +provisioner: openebs.io/local +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Delete +--- +# Create Maya Service Account +apiVersion: v1 +kind: ServiceAccount +metadata: + name: openebs-maya-operator + namespace: kube-system +--- +# Define Role that allows operations on K8s pods/deployments +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-maya-operator +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["*"] + - apiGroups: ["*"] + resources: ["nodes", "nodes/proxy"] + verbs: ["*"] + - apiGroups: ["*"] + resources: ["namespaces", "services", "pods", "pods/exec", "deployments", "deployments/finalizers", "replicationcontrollers", "replicasets", "events", "endpoints", "configmaps", "secrets", "jobs", "cronjobs"] + verbs: ["*"] + - apiGroups: ["*"] + resources: ["statefulsets", "daemonsets"] + verbs: ["*"] + - apiGroups: ["*"] + resources: ["resourcequotas", "limitranges"] + verbs: ["list", "watch"] + - apiGroups: ["*"] + resources: ["ingresses", "horizontalpodautoscalers", "verticalpodautoscalers", "poddisruptionbudgets", "certificatesigningrequests"] + verbs: ["list", "watch"] + - apiGroups: ["*"] + resources: ["storageclasses", "persistentvolumeclaims", "persistentvolumes"] + verbs: ["*"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: [ "get", "list", "create", "update", "delete", "patch"] + - apiGroups: ["openebs.io"] + resources: [ "*"] + verbs: ["*"] + - nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +# Bind the Service Account with the Role Privileges. +# TODO: Check if default account also needs to be there +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-maya-operator +subjects: + - kind: ServiceAccount + name: openebs-maya-operator + namespace: kube-system +roleRef: + kind: ClusterRole + name: openebs-maya-operator + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: openebs-localpv-provisioner + namespace: kube-system + labels: + name: openebs-localpv-provisioner + openebs.io/component-name: openebs-localpv-provisioner + openebs.io/version: 3.3.0 +spec: + selector: + matchLabels: + name: openebs-localpv-provisioner + openebs.io/component-name: openebs-localpv-provisioner + replicas: 1 + strategy: + type: Recreate + template: + metadata: + labels: + name: openebs-localpv-provisioner + openebs.io/component-name: openebs-localpv-provisioner + openebs.io/version: 3.3.0 + spec: + serviceAccountName: openebs-maya-operator + containers: + - name: openebs-provisioner-hostpath + imagePullPolicy: IfNotPresent + image: {{ .sc.local.provisioner_image.registry }}/{{ .sc.local.provisioner_image.repository }}:{{ .sc.local.provisioner_image.tag }} + env: + # OPENEBS_IO_K8S_MASTER enables openebs provisioner to connect to K8s + # based on this address. This is ignored if empty. + # This is supported for openebs provisioner version 0.5.2 onwards + #- name: OPENEBS_IO_K8S_MASTER + # value: "http://10.128.0.12:8080" + # OPENEBS_IO_KUBE_CONFIG enables openebs provisioner to connect to K8s + # based on this config. This is ignored if empty. + # This is supported for openebs provisioner version 0.5.2 onwards + #- name: OPENEBS_IO_KUBE_CONFIG + # value: "/home/ubuntu/.kube/config" + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OPENEBS_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + # OPENEBS_SERVICE_ACCOUNT provides the service account of this pod as + # environment variable + - name: OPENEBS_SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: OPENEBS_IO_ENABLE_ANALYTICS + value: "true" + - name: OPENEBS_IO_INSTALLER_TYPE + value: "openebs-operator-lite" + - name: OPENEBS_IO_HELPER_IMAGE + value: "{{ .sc.local.linux_utils_image.registry }}/{{ .sc.local.linux_utils_image.repository }}:{{ .sc.local.linux_utils_image.tag }}" + # LEADER_ELECTION_ENABLED is used to enable/disable leader election. By default + # leader election is enabled. + #- name: LEADER_ELECTION_ENABLED + # value: "true" + # OPENEBS_IO_IMAGE_PULL_SECRETS environment variable is used to pass the image pull secrets + # to the helper pod launched by local-pv hostpath provisioner + #- name: OPENEBS_IO_IMAGE_PULL_SECRETS + # value: "" + livenessProbe: + exec: + command: + - sh + - -c + - test $(pgrep -c "^provisioner-loc.*") = 1 + initialDelaySeconds: 30 + periodSeconds: 60 diff --git a/builtin/capkk/roles/precheck/env_check/defaults/main.yaml b/builtin/capkk/roles/precheck/env_check/defaults/main.yaml new file mode 100644 index 000000000..aaacae229 --- /dev/null +++ b/builtin/capkk/roles/precheck/env_check/defaults/main.yaml @@ -0,0 +1,29 @@ +cluster_require: + # the etcd sync duration for 99%.(unit ns) + etcd_disk_wal_fysnc_duration_seconds: 10000000 + allow_unsupported_distribution_setup: false + # support ubuntu, centos. + supported_os_distributions: + - ubuntu + - '"ubuntu"' + - centos + - '"centos"' + require_network_plugin: ['calico', 'flannel', 'cilium', 'hybridnet', 'kube-ovn'] + # the minimal version of kubernetes to be installed. + kube_version_min_required: v1.23.0 + # memory size for each kube_control_plane node.(unit kB) + # should be greater than or equal to minimal_master_memory_mb. + minimal_master_memory_mb: 10 + # memory size for each kube_worker node.(unit kB) + # should be greater than or equal to minimal_node_memory_mb. + minimal_node_memory_mb: 10 + require_etcd_deployment_type: ['internal','external'] + require_container_manager: ['docker', 'containerd'] + # the minimal required version of containerd to be installed. + containerd_min_version_required: v1.6.0 + supported_architectures: + - amd64 + - x86_64 + - arm64 + - aarch64 + min_kernel_version: 4.9.17 diff --git a/builtin/capkk/roles/precheck/env_check/tasks/cri.yaml b/builtin/capkk/roles/precheck/env_check/tasks/cri.yaml new file mode 100644 index 000000000..dcd7ce864 --- /dev/null +++ b/builtin/capkk/roles/precheck/env_check/tasks/cri.yaml @@ -0,0 +1,18 @@ +--- +- name: Stop if container manager is not docker or containerd + assert: + that: .cluster_require.require_container_manager | has .cri.container_manager + fail_msg: >- + the container manager:{{ .cri.container_manager }}, must be {{ .cluster_require.require_container_manager | toJson }} + run_once: true + when: cri.container_manager | empty | not + +- name: Ensure minimum containerd version + assert: + that: .containerd_version | semverCompare (printf ">=%s" .cluster_require.containerd_min_version_required) + fail_msg: >- + containerd_version is too low. Minimum version {{ .cluster_require.containerd_min_version_required }} + run_once: true + when: + - .containerd_version | empty | not + - .cri.container_manager | eq "containerd" diff --git a/builtin/capkk/roles/precheck/env_check/tasks/kubernetes.yaml b/builtin/capkk/roles/precheck/env_check/tasks/kubernetes.yaml new file mode 100644 index 000000000..849d925f0 --- /dev/null +++ b/builtin/capkk/roles/precheck/env_check/tasks/kubernetes.yaml @@ -0,0 +1,63 @@ +- name: Should defined internal_ipv4 or internal_ipv6 + assert: + that: or (.internal_ipv4 | empty | not) (.internal_ipv6 | empty | not)) + fail_msg: >- + "internal_ipv4" and "internal_ipv6" cannot both be empty + +- name: Check kubevip if valid + run_once: true + assert: + that: + - .kubernetes.control_plane_endpoint.kube_vip.address | empty | not + - .kubernetes.control_plane_endpoint.kube_vip.address | regexMatch "^((25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])|(([0-9a-fA-F]{1,4}:){7}([0-9a-fA-F]{1,4}|:)|(([0-9a-fA-F]{1,4}:){1,6}|:):([0-9a-fA-F]{1,4}|:){1,6}([0-9a-fA-F]{1,4}|:)))$" + - | + {{- $existIP := false }} + {{- range .groups.all | default list }} + {{- if eq $.kubernetes.control_plane_endpoint.kube_vip.address (index $.hostvars . "internal_ipv4") }} + {{- $existIP = true }} + {{- end }} + {{- end }} + {{ not $existIP }} + fail_msg: >- + "kubernetes.control_plane_endpoint.kube_vip.address" should be a un-used ip address. + when: .kubernetes.control_plane_endpoint.type | eq "kube_vip" + +- name: Stop if unsupported version of Kubernetes + run_once: true + assert: + that: .kube_version | semverCompare (printf ">=%s" .cluster_require.kube_version_min_required) + fail_msg: >- + the current release of KubeKey only support newer version of Kubernetes than {{ .cluster_require.kube_version_min_required }} - You are trying to apply {{ .kube_version }} + when: .kube_version | empty | not + +- name: Check if kubernetes installed + when: .groups.k8s_cluster | default list | has .inventory_hostname + block: + - name: Get kubelet.service LoadState and save to variable + command: systemctl show kubelet.service -p LoadState --value + register: kubernetes_install_LoadState + - name: Get kubelet.service ActiveState and save to variable + command: systemctl show kubelet.service -p ActiveState --value + register: kubernetes_install_ActiveState + - name: Get kubernetes version + ignore_errors: true + command: kubelet --version + register: kubernetes_install_version + - name: Check kubernetes service and version + when: .kubernetes_install_LoadState.stdout | eq "loaded" + block: + - name: Kubernetes should be active + assert: + that: .kubernetes_install_ActiveState.stdout | eq "active" + fail_msg: >- + kubernetes should be active when it's loaded + - name: Check kubernetes version + assert: + that: .kubernetes_install_version.stdout | default "" | trimPrefix "Kubernetes " | eq .kube_version + fail_msg: >- + kubernetes has installed with version:{{ .kubernetes_install_version.stdout | default "" | trimPrefix "Kubernetes " }}. but not match kube_version: {{ .kube_version }} + - name: add kubernetes_version variable to all hosts + add_hostvars: + hosts: all + vars: + kubernetes_installed: true diff --git a/builtin/capkk/roles/precheck/env_check/tasks/main.yaml b/builtin/capkk/roles/precheck/env_check/tasks/main.yaml new file mode 100644 index 000000000..f1dc3c316 --- /dev/null +++ b/builtin/capkk/roles/precheck/env_check/tasks/main.yaml @@ -0,0 +1,12 @@ +--- +- include_tasks: kubernetes.yaml + +- include_tasks: os.yaml + tags: ["os"] + +- include_tasks: network.yaml + tags: ["network"] + +- include_tasks: cri.yaml + tags: ["cri"] + diff --git a/builtin/capkk/roles/precheck/env_check/tasks/network.yaml b/builtin/capkk/roles/precheck/env_check/tasks/network.yaml new file mode 100644 index 000000000..ac4a2db1a --- /dev/null +++ b/builtin/capkk/roles/precheck/env_check/tasks/network.yaml @@ -0,0 +1,79 @@ +--- +- name: Should found network interface + command: | + {{- if .internal_ipv4 | empty | not }} + if [ ! ip -o addr show | grep -q {{ .internal_ipv4 }} ]; then + echo "No ipv4 network interface found" + exit 1 + fi + {{- end }} + {{- if .internal_ipv6 | empty | not }} + if [ ! ip -o addr show | grep -q {{ .internal_ipv6 }} ]; then + echo "No ipv6 network interface found" + exit 1 + fi + {{- end }} + +# https://kubernetes.io/docs/concepts/services-networking/dual-stack/ +- name: Stop if cidr is not valid + block: + - name: Stop if pod cidr is not valid + when: .kubernetes.networking.pod_cidr | empty | not + assert: + that: .kubernetes.networking.pod_cidr | splitList "," | len | ge 2 + fail_msg: >- + "kubernetes.networking.pod_cidr" should be ipv4_cidr/ipv6_cidr or ipv4_cidr,ipv6_cidr + - name: Stop if service cidr is not valid + when: .kubernetes.networking.service_cidr | empty | not + assert: + that: .kubernetes.networking.service_cidr | splitList "," | len | ge 2 + fail_msg: >- + "kubernetes.networking.service_cidr" should be ipv4_cidr/ipv6_cidr or ipv4_cidr,ipv6_cidr + - name: Stop if pod networking is not support dual-stack + when: + - .kubernetes.networking.pod_cidr | empty | not + - .kubernetes.networking.pod_cidr | splitList "," | len | eq 2 + assert: + that: + - .kube_version | semverCompare ">=v1.20.0" + - .kubernetes.networking.pod_cidr | splitList "," | first | ipFamily | eq "IPv4" + - .kubernetes.networking.pod_cidr | splitList "," | last | ipFamily | eq "IPv6" + fail_msg: >- + Kubernetes introduced support for pod dual-stack networking starting from version v1.20.0. + - name: Stop if service networking is not support dual-stack + when: + - .kubernetes.networking.service_cidr | empty | not + - .kubernetes.networking.service_cidr | splitList "," | len | eq 2 + assert: + that: + - .kube_version | semverCompare ">=v1.20.0" + - .kubernetes.networking.service_cidr | splitList "," | first | ipFamily | eq "IPv4" + - .kubernetes.networking.service_cidr | splitList "," | last | ipFamily | eq "IPv6" + fail_msg: >- + Kubernetes introduced support for service dual-stack networking starting from version v1.20.0. + +- name: Stop if unknown network plugin + assert: + that: .cluster_require.require_network_plugin | has .kubernetes.kube_network_plugin + fail_msg: >- + kube_network_plugin:"{{ .kubernetes.kube_network_plugin }}" is not supported + when: .kubernetes.kube_network_plugin | empty | not + +# This assertion will fail on the safe side: One can indeed schedule more pods +# on a node than the CIDR-range has space for when additional pods use the host +# network namespace. It is impossible to ascertain the number of such pods at +# provisioning time, so to establish a guarantee, we factor these out. +# NOTICE: the check blatantly ignores the inet6-case +- name: Guarantee that enough network address space is available for all pods + when: .groups.k8s_cluster | default list | has .inventory_hostname + block: + - name: Guarantee that enough ipv4 network address space is available for all pods + when: .kubernetes.networking.pod_cidr | default "10.233.64.0/18" | splitList "," | first | ipFamily | eq "IPv4" + assert: + that: le .kubernetes.kubelet.max_pods (sub (pow 2 (sub 32 .kubernetes.networking.ipv4_mask_size | default 24 | float64)) 2) + fail_msg: do not schedule more pods on a node than ipv4 inet addresses are available. + - name: Guarantee that enough ipv6 network address space is available for all pods + when: .kubernetes.networking.pod_cidr | default "10.233.64.0/18" | splitList "," | last | ipFamily | eq "IPv6" + assert: + that: le .kubernetes.kubelet.max_pods (sub (pow 2 (sub 128 .kubernetes.networking.ipv6_mask_size | default 64 | float64)) 2) + fail_msg: do not schedule more pods on a node than ipv6 inet addresses are available. diff --git a/builtin/capkk/roles/precheck/env_check/tasks/os.yaml b/builtin/capkk/roles/precheck/env_check/tasks/os.yaml new file mode 100644 index 000000000..4c3ed02b8 --- /dev/null +++ b/builtin/capkk/roles/precheck/env_check/tasks/os.yaml @@ -0,0 +1,33 @@ +--- +- name: Stop if bad hostname + assert: + that: .hostname | regexMatch "^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*$" + fail_msg: >- + "Hostname must consist of lower case alphanumeric characters, '.' or '-', and must start and end with an alphanumeric character" + +- name: Stop if the os does not support + assert: + that: or (.cluster_require.allow_unsupported_distribution_setup) (.cluster_require.supported_os_distributions | has .os.release.ID) + fail_msg: "{{ .os.release.ID }} is not a known OS" + +- name: Stop if arch supported + assert: + that: .cluster_require.supported_architectures | has .os.architecture + fail_msg: "{{ .os.architecture }} is not a known arch" + +- name: Stop if memory is too small for masters + assert: + that: .process.memInfo.MemTotal | trimSuffix " kB" | atoi | le .cluster_require.minimal_master_memory_mb + when: .groups.kube_control_plane | default list | has .inventory_hostname + +- name: Stop if memory is too small for nodes + assert: + that: .process.memInfo.MemTotal | trimSuffix " kB" | atoi | le .cluster_require.minimal_node_memory_mb + when: + - .groups.kube_worker | default list | has .inventory_hostname + +- name: Stop if kernel version is too low + assert: + that: .os.kernel_version | splitList "-" | first | semverCompare (printf ">=%s" .cluster_require.min_kernel_version) + fail_msg: >- + "kernel version: {{ .os.kernel_version }} is too low, required at least: {{ .cluster_require.min_kernel_version }} " diff --git a/builtin/capkk/roles/uninstall/cloud-config/tasks/main.yaml b/builtin/capkk/roles/uninstall/cloud-config/tasks/main.yaml new file mode 100644 index 000000000..b2f9deade --- /dev/null +++ b/builtin/capkk/roles/uninstall/cloud-config/tasks/main.yaml @@ -0,0 +1,8 @@ +--- +- name: deal cloud-config users + loop: "{{ .cloud_config.users | toJson }}" + command: | + #!/bin/bash + + # Delete user '{{ .item.name }}' + userdel -r "{{ .item.name }}" diff --git a/builtin/capkk/roles/uninstall/cri/defaults/main.yaml b/builtin/capkk/roles/uninstall/cri/defaults/main.yaml new file mode 100644 index 000000000..742df21b5 --- /dev/null +++ b/builtin/capkk/roles/uninstall/cri/defaults/main.yaml @@ -0,0 +1,5 @@ +cri: + containerd: + data_root: /var/lib/containerd + docker: + data_root: /var/lib/docker \ No newline at end of file diff --git a/builtin/capkk/roles/uninstall/cri/tasks/main.yaml b/builtin/capkk/roles/uninstall/cri/tasks/main.yaml new file mode 100644 index 000000000..c761abfb1 --- /dev/null +++ b/builtin/capkk/roles/uninstall/cri/tasks/main.yaml @@ -0,0 +1,18 @@ +--- +- name: Stop containerd + include_tasks: uninstall_containerd.yaml + when: .cri.container_manager | eq "containerd" + +- name: Stop docker + include_tasks: uninstall_docker.yaml + when: .cri.container_manager | eq "docker" + +# install cridockerd +- include_tasks: uninstall_cridockerd.yaml + when: + - .cri.container_manager | eq "docker" + - .cridockerd_version | empty | not + +- name: Delete cri residue files + command: | + rm -f /usr/local/bin/crictl \ No newline at end of file diff --git a/builtin/capkk/roles/uninstall/cri/tasks/uninstall_containerd.yaml b/builtin/capkk/roles/uninstall/cri/tasks/uninstall_containerd.yaml new file mode 100644 index 000000000..6adb10454 --- /dev/null +++ b/builtin/capkk/roles/uninstall/cri/tasks/uninstall_containerd.yaml @@ -0,0 +1,17 @@ +--- +- name: Uninstall containerd service + ignore_errors: true + command: | + systemctl stop containerd.service + systemctl disable containerd.service + rm -rf /etc/systemd/system/containerd.service* + systemctl daemon-reload + systemctl reset-failed containerd.service + +- name: Delete containerd residue files + command: | + rm -rf {{ .cri.containerd.data_root }} + rm -rf /etc/containerd + rm -rf /usr/local/bin/containerd* + rm -f /usr/local/bin/runc + rm -f /usr/local/bin/ctr diff --git a/builtin/capkk/roles/uninstall/cri/tasks/uninstall_cridockerd.yaml b/builtin/capkk/roles/uninstall/cri/tasks/uninstall_cridockerd.yaml new file mode 100644 index 000000000..74c00965e --- /dev/null +++ b/builtin/capkk/roles/uninstall/cri/tasks/uninstall_cridockerd.yaml @@ -0,0 +1,14 @@ +--- +- name: Stop cri-dockerd service + ignore_errors: true + command: | + systemctl stop cri-dockerd.service + systemctl disable cri-dockerd.service + rm -rf /etc/systemd/system/cri-dockerd.service* + systemctl daemon-reload + systemctl reset-failed cri-dockerd.service + +- name: Delete cri-dockerd residue files + command: | + rm -rf /etc/cri-dockerd + rm -f /usr/local/bin/cri-dockerd \ No newline at end of file diff --git a/builtin/capkk/roles/uninstall/cri/tasks/uninstall_docker.yaml b/builtin/capkk/roles/uninstall/cri/tasks/uninstall_docker.yaml new file mode 100644 index 000000000..b223e69cb --- /dev/null +++ b/builtin/capkk/roles/uninstall/cri/tasks/uninstall_docker.yaml @@ -0,0 +1,22 @@ +--- +- name: Stop docker service + ignore_errors: true + command: | + systemctl stop docker.service + systemctl disable docker.service + rm -rf /etc/systemd/system/docker.service* + systemctl daemon-reload + systemctl reset-failed docker.service + +- name: Uninstall containerd + include_tasks: uninstall_containerd.yaml + +- name: Delete docker residue files + command: | + rm -rf {{ .cri.docker.data_root }} + rm -rf /etc/docker + rm -rf /usr/local/bin/docker* + +- name: Uninstall docker interface + ignore_errors: true + command: ip link delete docker0 \ No newline at end of file diff --git a/builtin/capkk/roles/uninstall/kubernetes/defaults/main.yaml b/builtin/capkk/roles/uninstall/kubernetes/defaults/main.yaml new file mode 100644 index 000000000..5e105ddeb --- /dev/null +++ b/builtin/capkk/roles/uninstall/kubernetes/defaults/main.yaml @@ -0,0 +1,3 @@ +cni: + type: >- + {{ .kubernetes.kube_network_plugin | default "calico" }} \ No newline at end of file diff --git a/builtin/capkk/roles/uninstall/kubernetes/tasks/kubernetes.yaml b/builtin/capkk/roles/uninstall/kubernetes/tasks/kubernetes.yaml new file mode 100644 index 000000000..e68ae092c --- /dev/null +++ b/builtin/capkk/roles/uninstall/kubernetes/tasks/kubernetes.yaml @@ -0,0 +1,33 @@ +--- +- name: Delete Node + command: | + if kubectl get node {{ .hostname }} > /dev/null 2>&1; then + kubectl cordon {{ .hostname }} + if [ $(kubectl get nodes --no-headers | wc -l) -gt 1 ]; then + kubectl drain {{ .hostname }} --ignore-daemonsets --delete-emptydir-data --force --disable-eviction + else + kubectl drain {{ .hostname }} --ignore-daemonsets --delete-emptydir-data --force + fi + {{- if .cni.type | eq "calico" }} + calicoctl delete node {{ .hostname }} + {{- end }} + kubectl delete node {{ .hostname }} + kubeadm reset -f + fi + +- name: Stop kubelet service + ignore_errors: true + command: | + systemctl stop kubelet.service + systemctl disable kubelet.service + rm -rf /etc/systemd/system/kubelet.service* + systemctl daemon-reload + systemctl reset-failed kubelet.service + +- name: Delete residue files + command: | + rm -rf /usr/local/bin/kubeadm && rm -rf /usr/local/bin/kubelet && rm -rf /usr/local/bin/kubectl + rm -rf /var/lib/kubelet/ + rm -rf /etc/kubernetes/ + rm -rf .kube/config + rm -rf /var/lib/etcd \ No newline at end of file diff --git a/builtin/capkk/roles/uninstall/kubernetes/tasks/main.yaml b/builtin/capkk/roles/uninstall/kubernetes/tasks/main.yaml new file mode 100644 index 000000000..33acd7181 --- /dev/null +++ b/builtin/capkk/roles/uninstall/kubernetes/tasks/main.yaml @@ -0,0 +1,8 @@ +--- +- include_tasks: kubernetes.yaml + +- include_tasks: network.yaml + +- name: Delete residue files + command: | + rm -rf /etc/kubekey/ \ No newline at end of file diff --git a/builtin/capkk/roles/uninstall/kubernetes/tasks/network.yaml b/builtin/capkk/roles/uninstall/kubernetes/tasks/network.yaml new file mode 100644 index 000000000..ea1b9d337 --- /dev/null +++ b/builtin/capkk/roles/uninstall/kubernetes/tasks/network.yaml @@ -0,0 +1,39 @@ +--- +- name: Reset iptables + command: | + iptables -F + iptables -X + iptables -F -t nat + iptables -X -t nat + ipvsadm -C + ip link del kube-ipvs0 + ip link del nodelocaldns + ip link del cni0 + {{- if .cni.type | eq "flannel" }} + ip link del flannel.1 + ip link del flannel-v6.1 + ip link del flannel-wg + ip link del flannel-wg-v6 + {{- end }} + {{- if .cni.type | eq "cilium" }} + ip link del cilium_host + ip link del cilium_vxlan + {{- end }} + {{- if .cni.type | eq "calico" }} + ip -br link show | grep cali[a-f0-9]* | awk -F @ '{print $1}' | xargs -r -t -n 1 ip link del + {{- end }} + ip netns show 2>/dev/null | grep cni- | awk '{print $1}' | xargs -r -t -n 1 ip netns del + +- name: Delete net.d + command: | + rm -rf /etc/cni/net.d/ + rm -rf /var/lib/cni/ + {{- if .cni.type | eq "calico" }} + rm -rf /usr/local/bin/calicoctl + {{- end }} + +- name: Delete arp by kube-vip + when: eq .kubernetes.control_plane_endpoint.type "kube_vip" + command: | + ip neigh show | grep {{ .kubernetes.control_plane_endpoint.kube_vip.address }} | awk '{print $1 " dev " $3}' | xargs -r -L1 ip neigh delete + ip -o addr show | grep {{ .kubernetes.control_plane_endpoint.kube_vip.address }} | awk '{system("ip addr del "$4" dev "$2)}' diff --git a/builtin/core/defaults/config/v1.23.yaml b/builtin/core/defaults/config/v1.23.yaml new file mode 100644 index 000000000..e162de656 --- /dev/null +++ b/builtin/core/defaults/config/v1.23.yaml @@ -0,0 +1,103 @@ +apiVersion: kubekey.kubesphere.io/v1 +kind: Config +spec: + download: + # if set as "cn", so that online downloads will try to use available domestic sources whenever possible. + zone: "" + kubernetes: + kube_version: {{ .kube_version }} + # helm binary + helm_version: v3.8.2 + etcd: + # etcd binary + etcd_version: v3.5.4 + image_registry: + # keepalived image tag. Used for load balancing when there are multiple image registry nodes. + keepalived_version: 2.0.20 + # ========== image registry: harbor ========== + # harbor image tag + harbor_version: v2.6.3 + # docker-compose binary + dockercompose_version: v2.12.2 + # ========== image registry: docker-registry ========== + # docker-registry image tag + docker_registry_version: 2.8.3 + cri: + # support: containerd,docker + container_manager: docker + sandbox_image: + tag: "3.6" + # ========== cri ========== + # crictl binary + crictl_version: v1.23.0 + # ========== cri: docker ========== + # docker binary + docker_version: 20.10.18 + # cridockerd. Required when kube_version is greater than 1.24 + cridockerd_version: v0.3.10 + # ========== cri: containerd ========== + # containerd binary + containerd_version: v1.6.8 + # runc binary + runc_version: v1.1.4 + cni: + multus: + image: + tag: v3.9.3 + # ========== cni ========== + # cni_plugins binary (optional) + # cni_plugins_version: v1.1.1 + # ========== cni: calico ========== + # calicoctl binary + calico_version: v3.24.5 + # ========== cni: cilium ========== + # cilium helm + cilium_version: 1.12.6 + # ========== cni: kubeovn ========== + # kubeovn helm + kubeovn_version: 1.10.0 + # ========== cni: hybridnet ========== + # hybridnet helm + hybridnet_version: 0.6.8 + storage_class: + # ========== storageclass ========== + # ========== storageclass: local ========== + local: + provisioner_image: + tag: 3.3.0 + linux_utils_image: + tag: 3.3.0 + # ========== storageclass: nfs ========== + # nfs provisioner helm version + nfs_provisioner_version: 4.0.2 + dns: + dns_image: + tag: v1.8.6 + dns_cache_image: + tag: 1.21.1 + image_manifests: + - docker.io/calico/apiserver:v3.24.5 + - docker.io/calico/cni:v3.24.5 + - docker.io/calico/csi:v3.24.5 + - docker.io/calico/kube-controllers:v3.24.5 + - docker.io/calico/node-driver-registrar:v3.24.5 + - docker.io/calico/node:v3.24.5 + - docker.io/calico/pod2daemon-flexvol:v3.24.5 + - docker.io/kubesphere/k8s-dns-node-cache:1.21.1 + - docker.io/openebs/provisioner-localpv:3.3.0 + - docker.io/kubesphere/coredns:v1.8.6 + - docker.io/kubesphere/kube-apiserver:{{ .kube_version }} + - docker.io/kubesphere/kube-controller-manager:{{ .kube_version }} + - docker.io/kubesphere/kube-proxy:{{ .kube_version }} + - docker.io/kubesphere/kube-scheduler:{{ .kube_version }} + - docker.io/kubesphere/pause:3.6 + - quay.io/tigera/operator:v1.28.5 + - docker.io/calico/ctl:v3.24.5 + - docker.io/calico/typha:v3.24.5 + - docker.io/calico/apiserver:v3.24.5 + - docker.io/calico/kube-controllers:v3.24.5 + - docker.io/calico/node:v3.24.5 + - docker.io/calico/pod2daemon-flexvol:v3.24.5 + - docker.io/calico/cni:v3.24.5 + - docker.io/calico/node-driver-registrar:v3.24.5 + - docker.io/calico/csi:v3.24.5 diff --git a/builtin/core/defaults/config/v1.24.yaml b/builtin/core/defaults/config/v1.24.yaml new file mode 100644 index 000000000..5d143a7aa --- /dev/null +++ b/builtin/core/defaults/config/v1.24.yaml @@ -0,0 +1,104 @@ +apiVersion: kubekey.kubesphere.io/v1 +kind: Config +spec: + download: + # if set as "cn", so that online downloads will try to use available domestic sources whenever possible. + zone: "" + kubernetes: + kube_version: {{ .kube_version }} + # helm binary + helm_version: v3.10.3 + etcd: + # etcd binary + etcd_version: v3.5.6 + image_registry: + # ========== image registry ========== + # keepalived image tag. Used for load balancing when there are multiple image registry nodes. + keepalived_version: v2.0.20 + # ========== image registry: harbor ========== + # harbor image tag + harbor_version: v2.7.1 + # docker-compose binary + dockercompose_version: v2.14.0 + # ========== image registry: docker-registry ========== + # docker-registry image tag + docker_registry_version: 2.8.3 + cri: + # support: containerd,docker + container_manager: containerd + sandbox_image: + tag: "3.6" + # ========== cri ========== + # crictl binary + crictl_version: v1.24.0 + # ========== cri: docker ========== + # docker binary + docker_version: 20.10.24 + # cridockerd. Required when kube_version is greater than 1.24 + cridockerd_version: v0.3.1 + # ========== cri: containerd ========== + # containerd binary + containerd_version: v1.6.16 + # runc binary + runc_version: v1.1.4 + cni: + multus: + image: + tag: v3.10.1 + # ========== cni ========== + # cni_plugins binary (optional) + # cni_plugins_version: v1.1.1 + # ========== cni: calico ========== + # calicoctl binary + calico_version: v3.25.1 + # ========== cni: cilium ========== + # cilium helm + cilium_version: 1.13.5 + # ========== cni: kubeovn ========== + # kubeovn helm + kubeovn_version: 1.10.0 + # ========== cni: hybridnet ========== + # hybridnet helm + hybridnet_version: 0.6.8 + storage_class: + # ========== storageclass ========== + # ========== storageclass: local ========== + local: + provisioner_image: + tag: 3.4.0 + linux_utils_image: + tag: 3.4.0 + # ========== storageclass: nfs ========== + # nfs provisioner helm version + nfs_provisioner_version: 4.0.2 + dns: + dns_image: + tag: v1.8.6 + dns_cache_image: + tag: 1.22.20 + image_manifests: + - docker.io/calico/apiserver:v3.25.1 + - docker.io/calico/cni:v3.25.1 + - docker.io/calico/csi:v3.25.1 + - docker.io/calico/kube-controllers:v3.25.1 + - docker.io/calico/node-driver-registrar:v3.25.1 + - docker.io/calico/node:v3.25.1 + - docker.io/calico/pod2daemon-flexvol:v3.25.1 + - docker.io/kubesphere/k8s-dns-node-cache:1.22.20 + - docker.io/openebs/provisioner-localpv:3.4.0 + - docker.io/kubesphere/coredns:v1.8.6 + - docker.io/kubesphere/kube-apiserver:{{ .kube_version }} + - docker.io/kubesphere/kube-controller-manager:{{ .kube_version }} + - docker.io/kubesphere/kube-proxy:{{ .kube_version }} + - docker.io/kubesphere/kube-scheduler:{{ .kube_version }} + - docker.io/kubesphere/pause:3.6 + - quay.io/tigera/operator:v1.29.3 + - docker.io/calico/ctl:v3.25.1 + - docker.io/calico/typha:v3.25.1 + - docker.io/calico/apiserver:v3.25.1 + - docker.io/calico/kube-controllers:v3.25.1 + - docker.io/calico/node:v3.25.1 + - docker.io/calico/pod2daemon-flexvol:v3.25.1 + - docker.io/calico/cni:v3.25.1 + - docker.io/calico/node-driver-registrar:v3.25.1 + - docker.io/calico/csi:v3.25.1 diff --git a/builtin/core/defaults/config/v1.25.yaml b/builtin/core/defaults/config/v1.25.yaml new file mode 100644 index 000000000..3241d3176 --- /dev/null +++ b/builtin/core/defaults/config/v1.25.yaml @@ -0,0 +1,104 @@ +apiVersion: kubekey.kubesphere.io/v1 +kind: Config +spec: + download: + # if set as "cn", so that online downloads will try to use available domestic sources whenever possible. + zone: "" + kubernetes: + kube_version: {{ .kube_version }} + # helm binary + helm_version: v3.10.3 + etcd: + # etcd binary + etcd_version: v3.5.7 + image_registry: + # ========== image registry ========== + # keepalived image tag. Used for load balancing when there are multiple image registry nodes. + keepalived_version: 2.0.20 + # ========== image registry: harbor ========== + # harbor image tag + harbor_version: v2.8.1 + # docker-compose binary + dockercompose_version: v2.15.1 + # ========== image registry: docker-registry ========== + # docker-registry image tag + docker_registry_version: 2.8.3 + cri: + # support: containerd,docker + container_manager: containerd + sandbox_image: + tag: "3.6" + # ========== cri ========== + # crictl binary + crictl_version: v1.25.0 + # ========== cri: docker ========== + # docker binary + docker_version: 20.10.24 + # cridockerd. Required when kube_version is greater than 1.24 + cridockerd_version: v0.3.1 + # ========== cri: containerd ========== + # containerd binary + containerd_version: v1.6.19 + # runc binary + runc_version: v1.1.4 + cni: + multus: + image: + tag: v3.11.3 + # ========== cni ========== + # cni_plugins binary (optional) + # cni_plugins_version: v1.1.1 + # ========== cni: calico ========== + # calicoctl binary + calico_version: v3.25.1 + # ========== cni: cilium ========== + # cilium helm + cilium_version: 1.13.5 + # ========== cni: kubeovn ========== + # kubeovn helm + kubeovn_version: 1.10.0 + # ========== cni: hybridnet ========== + # hybridnet helm + hybridnet_version: 0.6.8 + storage_class: + # ========== storageclass ========== + # ========== storageclass: local ========== + local: + provisioner_image: + tag: 3.4.0 + linux_utils_image: + tag: 3.4.0 + # ========== storageclass: nfs ========== + # nfs provisioner helm version + nfs_provisioner_version: 4.0.8 + dns: + dns_image: + tag: v1.9.3 + dns_cache_image: + tag: 1.22.20 + image_manifests: + - docker.io/calico/apiserver:v3.25.1 + - docker.io/calico/cni:v3.25.1 + - docker.io/calico/csi:v3.25.1 + - docker.io/calico/kube-controllers:v3.25.1 + - docker.io/calico/node-driver-registrar:v3.25.1 + - docker.io/calico/node:v3.25.1 + - docker.io/calico/pod2daemon-flexvol:v3.25.1 + - docker.io/kubesphere/k8s-dns-node-cache:1.22.20 + - docker.io/openebs/provisioner-localpv:3.4.0 + - docker.io/kubesphere/coredns:v1.9.3 + - docker.io/kubesphere/kube-apiserver:{{ .kube_version }} + - docker.io/kubesphere/kube-controller-manager:{{ .kube_version }} + - docker.io/kubesphere/kube-proxy:{{ .kube_version }} + - docker.io/kubesphere/kube-scheduler:{{ .kube_version }} + - docker.io/kubesphere/pause:3.6 + - quay.io/tigera/operator:v1.29.3 + - docker.io/calico/ctl:v3.25.1 + - docker.io/calico/typha:v3.25.1 + - docker.io/calico/apiserver:v3.25.1 + - docker.io/calico/kube-controllers:v3.25.1 + - docker.io/calico/node:v3.25.1 + - docker.io/calico/pod2daemon-flexvol:v3.25.1 + - docker.io/calico/cni:v3.25.1 + - docker.io/calico/node-driver-registrar:v3.25.1 + - docker.io/calico/csi:v3.25.1 diff --git a/builtin/core/defaults/config/v1.26.yaml b/builtin/core/defaults/config/v1.26.yaml new file mode 100644 index 000000000..e10139d80 --- /dev/null +++ b/builtin/core/defaults/config/v1.26.yaml @@ -0,0 +1,104 @@ +apiVersion: kubekey.kubesphere.io/v1 +kind: Config +spec: + download: + # if set as "cn", so that online downloads will try to use available domestic sources whenever possible. + zone: "" + kubernetes: + kube_version: {{ .kube_version }} + # helm binary + helm_version: v3.11.2 + etcd: + # etcd binary + etcd_version: v3.5.8 + image_registry: + # ========== image registry ========== + # keepalived image tag. Used for load balancing when there are multiple image registry nodes. + keepalived_version: 2.0.20 + # ========== image registry: harbor ========== + # harbor image tag + harbor_version: v2.9.1 + # docker-compose binary + dockercompose_version: v2.16.0 + # ========== image registry: docker-registry ========== + # docker-registry image tag + docker_registry_version: 2.8.3 + cri: + # support: containerd,docker + container_manager: containerd + sandbox_image: + tag: "3.7" + # ========== cri ========== + # crictl binary + crictl_version: v1.26.0 + # ========== cri: docker ========== + # docker binary + docker_version: 23.0.6 + # cridockerd. Required when kube_version is greater than 1.24 + cridockerd_version: v0.3.1 + # ========== cri: containerd ========== + # containerd binary + containerd_version: v1.6.21 + # runc binary + runc_version: v1.1.5 + cni: + multus: + image: + tag: v4.0.2 + # ========== cni ========== + # cni_plugins binary (optional) + # cni_plugins_version: v1.2.0 + # ========== cni: calico ========== + # calicoctl binary + calico_version: v3.26.1 + # ========== cni: cilium ========== + # cilium helm + cilium_version: 1.13.5 + # ========== cni: kubeovn ========== + # kubeovn helm + kubeovn_version: 1.10.0 + # ========== cni: hybridnet ========== + # hybridnet helm + hybridnet_version: 0.6.8 + storage_class: + # ========== storageclass ========== + # ========== storageclass: local ========== + local: + provisioner_image: + tag: 3.4.0 + linux_utils_image: + tag: 3.4.0 + # ========== storageclass: nfs ========== + # nfs provisioner helm version + nfs_provisioner_version: 4.0.8 + dns: + dns_image: + tag: v1.9.3 + dns_cache_image: + tag: 1.22.20 + image_manifests: + - docker.io/calico/apiserver:v3.26.1 + - docker.io/calico/cni:v3.26.1 + - docker.io/calico/csi:v3.26.1 + - docker.io/calico/kube-controllers:v3.26.1 + - docker.io/calico/node-driver-registrar:v3.26.1 + - docker.io/calico/node:v3.26.1 + - docker.io/calico/pod2daemon-flexvol:v3.26.1 + - docker.io/kubesphere/k8s-dns-node-cache:1.22.20 + - docker.io/openebs/provisioner-localpv:3.4.0 + - docker.io/kubesphere/coredns:v1.9.3 + - docker.io/kubesphere/kube-apiserver:{{ .kube_version }} + - docker.io/kubesphere/kube-controller-manager:{{ .kube_version }} + - docker.io/kubesphere/kube-proxy:{{ .kube_version }} + - docker.io/kubesphere/kube-scheduler:{{ .kube_version }} + - docker.io/kubesphere/pause:3.7 + - quay.io/tigera/operator:v1.30.4 + - docker.io/calico/ctl:v3.26.1 + - docker.io/calico/typha:v3.26.1 + - docker.io/calico/apiserver:v3.26.1 + - docker.io/calico/kube-controllers:v3.26.1 + - docker.io/calico/node:v3.26.1 + - docker.io/calico/pod2daemon-flexvol:v3.26.1 + - docker.io/calico/cni:v3.26.1 + - docker.io/calico/node-driver-registrar:v3.26.1 + - docker.io/calico/csi:v3.26.1 diff --git a/builtin/core/defaults/config/v1.27.yaml b/builtin/core/defaults/config/v1.27.yaml new file mode 100644 index 000000000..17b6fa55c --- /dev/null +++ b/builtin/core/defaults/config/v1.27.yaml @@ -0,0 +1,104 @@ +apiVersion: kubekey.kubesphere.io/v1 +kind: Config +spec: + download: + # if set as "cn", so that online downloads will try to use available domestic sources whenever possible. + zone: "" + kubernetes: + kube_version: {{ .kube_version }} + # helm binary + helm_version: v3.12.1 + etcd: + # etcd binary + etcd_version: v3.5.9 + image_registry: + # ========== image registry ========== + # keepalived image tag. Used for load balancing when there are multiple image registry nodes. + keepalived_version: 2.0.20 + # ========== image registry: harbor ========== + # harbor image tag + harbor_version: v2.10.1 + # docker-compose binary + dockercompose_version: v2.20.3 + # ========== image registry: docker-registry ========== + # docker-registry image tag + docker_registry_version: 2.8.3 + cri: + # support: containerd,docker + container_manager: containerd + sandbox_image: + tag: "3.7" + # ========== cri ========== + # crictl binary + crictl_version: v1.27.0 + # ========== cri: docker ========== + # docker binary + docker_version: 23.0.6 + # cridockerd. Required when kube_version is greater than 1.24 + cridockerd_version: v0.3.1 + # ========== cri: containerd ========== + # containerd binary + containerd_version: v1.7.2 + # runc binary + runc_version: v1.1.7 + cni: + multus: + image: + tag: v4.0.2 + # ========== cni ========== + # cni_plugins binary (optional) + # cni_plugins_version: v1.2.0 + # ========== cni: calico ========== + # calicoctl binary + calico_version: v3.26.1 + # ========== cni: cilium ========== + # cilium helm + cilium_version: 1.14.2 + # ========== cni: kubeovn ========== + # kubeovn helm + kubeovn_version: 1.11.0 + # ========== cni: hybridnet ========== + # hybridnet helm + hybridnet_version: 0.6.8 + storage_class: + # ========== storageclass ========== + # ========== storageclass: local ========== + local: + provisioner_image: + tag: 3.4.0 + linux_utils_image: + tag: 3.4.0 + # ========== storageclass: nfs ========== + # nfs provisioner helm version + nfs_provisioner_version: 4.0.10 + dns: + dns_image: + tag: v1.10.1 + dns_cache_image: + tag: 1.22.20 + image_manifests: + - docker.io/calico/apiserver:v3.26.1 + - docker.io/calico/cni:v3.26.1 + - docker.io/calico/csi:v3.26.1 + - docker.io/calico/kube-controllers:v3.26.1 + - docker.io/calico/node-driver-registrar:v3.26.1 + - docker.io/calico/node:v3.26.1 + - docker.io/calico/pod2daemon-flexvol:v3.26.1 + - docker.io/kubesphere/k8s-dns-node-cache:1.22.20 + - docker.io/openebs/provisioner-localpv:3.4.0 + - docker.io/kubesphere/coredns:v1.10.1 + - docker.io/kubesphere/kube-apiserver:{{ .kube_version }} + - docker.io/kubesphere/kube-controller-manager:{{ .kube_version }} + - docker.io/kubesphere/kube-proxy:{{ .kube_version }} + - docker.io/kubesphere/kube-scheduler:{{ .kube_version }} + - docker.io/kubesphere/pause:3.7 + - quay.io/tigera/operator:v1.30.4 + - docker.io/calico/ctl:v3.26.1 + - docker.io/calico/typha:v3.26.1 + - docker.io/calico/apiserver:v3.26.1 + - docker.io/calico/kube-controllers:v3.26.1 + - docker.io/calico/node:v3.26.1 + - docker.io/calico/pod2daemon-flexvol:v3.26.1 + - docker.io/calico/cni:v3.26.1 + - docker.io/calico/node-driver-registrar:v3.26.1 + - docker.io/calico/csi:v3.26.1 diff --git a/builtin/core/defaults/config/v1.28.yaml b/builtin/core/defaults/config/v1.28.yaml new file mode 100644 index 000000000..8f3266916 --- /dev/null +++ b/builtin/core/defaults/config/v1.28.yaml @@ -0,0 +1,104 @@ +apiVersion: kubekey.kubesphere.io/v1 +kind: Config +spec: + download: + # if set as "cn", so that online downloads will try to use available domestic sources whenever possible. + zone: "" + kubernetes: + kube_version: {{ .kube_version }} + # helm binary + helm_version: v3.12.1 + etcd: + # etcd binary + etcd_version: v3.5.9 + image_registry: + # ========== image registry ========== + # keepalived image tag. Used for load balancing when there are multiple image registry nodes. + keepalived_version: 2.0.20 + # ========== image registry: harbor ========== + # harbor image tag + harbor_version: v2.10.1 + # docker-compose binary + dockercompose_version: v2.20.3 + # ========== image registry: docker-registry ========== + # docker-registry image tag + docker_registry_version: 2.8.3 + cri: + # support: containerd,docker + container_manager: containerd + sandbox_image: + tag: "3.8" + # ========== cri ========== + # crictl binary + crictl_version: v1.28.0 + # ========== cri: docker ========== + # docker binary + docker_version: 24.0.6 + # cridockerd. Required when kube_version is greater than 1.24 + cridockerd_version: v0.3.1 + # ========== cri: containerd ========== + # containerd binary + containerd_version: v1.7.3 + # runc binary + runc_version: v1.1.7 + cni: + multus: + image: + tag: v4.1.0 + # ========== cni ========== + # cni_plugins binary (optional) + # cni_plugins_version: v1.2.0 + # ========== cni: calico ========== + # calicoctl binary + calico_version: v3.28.2 + # ========== cni: cilium ========== + # cilium helm + cilium_version: 1.15.0 + # ========== cni: kubeovn ========== + # kubeovn helm + kubeovn_version: 1.12.0 + # ========== cni: hybridnet ========== + # hybridnet helm + hybridnet_version: 0.6.8 + storage_class: + # ========== storageclass ========== + # ========== storageclass: local ========== + local: + provisioner_image: + tag: 3.4.0 + linux_utils_image: + tag: 3.4.0 + # ========== storageclass: nfs ========== + # nfs provisioner helm version + nfs_provisioner_version: 4.0.10 + dns: + dns_image: + tag: v1.10.1 + dns_cache_image: + tag: 1.22.20 + image_manifests: + - docker.io/calico/apiserver:v3.28.2 + - docker.io/calico/cni:v3.28.2 + - docker.io/calico/csi:v3.28.2 + - docker.io/calico/kube-controllers:v3.28.2 + - docker.io/calico/node-driver-registrar:v3.28.2 + - docker.io/calico/node:v3.28.2 + - docker.io/calico/pod2daemon-flexvol:v3.28.2 + - docker.io/kubesphere/k8s-dns-node-cache:1.22.20 + - docker.io/openebs/provisioner-localpv:3.4.0 + - docker.io/kubesphere/coredns:v1.10.1 + - docker.io/kubesphere/kube-apiserver:{{ .kube_version }} + - docker.io/kubesphere/kube-controller-manager:{{ .kube_version }} + - docker.io/kubesphere/kube-proxy:{{ .kube_version }} + - docker.io/kubesphere/kube-scheduler:{{ .kube_version }} + - docker.io/kubesphere/pause:3.8 + - quay.io/tigera/operator:v1.34.5 + - docker.io/calico/ctl:v3.28.2 + - docker.io/calico/typha:v3.28.2 + - docker.io/calico/apiserver:v3.28.2 + - docker.io/calico/kube-controllers:v3.28.2 + - docker.io/calico/node:v3.28.2 + - docker.io/calico/pod2daemon-flexvol:v3.28.2 + - docker.io/calico/cni:v3.28.2 + - docker.io/calico/node-driver-registrar:v3.28.2 + - docker.io/calico/csi:v3.28.2 diff --git a/builtin/core/defaults/config/v1.29.yaml b/builtin/core/defaults/config/v1.29.yaml new file mode 100644 index 000000000..bba165430 --- /dev/null +++ b/builtin/core/defaults/config/v1.29.yaml @@ -0,0 +1,104 @@ +apiVersion: kubekey.kubesphere.io/v1 +kind: Config +spec: + download: + # if set as "cn", so that online downloads will try to use available domestic sources whenever possible. + zone: "" + kubernetes: + kube_version: {{ .kube_version }} + # helm binary + helm_version: v3.13.3 + etcd: + # etcd binary + etcd_version: v3.5.10 + image_registry: + # ========== image registry ========== + # keepalived image tag. Used for load balancing when there are multiple image registry nodes. + keepalived_version: 2.0.20 + # ========== image registry: harbor ========== + # harbor image tag + harbor_version: v2.10.1 + # docker-compose binary + dockercompose_version: v2.20.3 + # ========== image registry: docker-registry ========== + # docker-registry image tag + docker_registry_version: 2.8.3 + cri: + # support: containerd,docker + container_manager: containerd + sandbox_image: + tag: "3.8" + # ========== cri ========== + # crictl binary + crictl_version: v1.29.0 + # ========== cri: docker ========== + # docker binary + docker_version: 24.0.7 + # cridockerd. Required when kube_version is greater than 1.24 + cridockerd_version: v0.3.1 + # ========== cri: containerd ========== + # containerd binary + containerd_version: v1.7.6 + # runc binary + runc_version: v1.1.7 + cni: + multus: + image: + tag: v4.1.1 + # ========== cni ========== + # cni_plugins binary (optional) + # cni_plugins_version: v1.2.0 + # ========== cni: calico ========== + # calicoctl binary + calico_version: v3.28.2 + # ========== cni: cilium ========== + # cilium helm + cilium_version: 1.15.4 + # ========== cni: kubeovn ========== + # kubeovn helm + kubeovn_version: 1.13.0 + # ========== cni: hybridnet ========== + # hybridnet helm + hybridnet_version: 0.6.8 + storage_class: + # ========== storageclass ========== + # ========== storageclass: local ========== + local: + provisioner_image: + tag: 3.5.0 + linux_utils_image: + tag: 3.5.0 + # ========== storageclass: nfs ========== + # nfs provisioner helm version + nfs_provisioner_version: 4.1.0 + dns: + dns_image: + tag: v1.11.1 + dns_cache_image: + tag: 1.23.1 + image_manifests: + - docker.io/calico/apiserver:v3.28.2 + - docker.io/calico/cni:v3.28.2 + - docker.io/calico/csi:v3.28.2 + - docker.io/calico/kube-controllers:v3.28.2 + - docker.io/calico/node-driver-registrar:v3.28.2 + - docker.io/calico/node:v3.28.2 + - docker.io/calico/pod2daemon-flexvol:v3.28.2 + - docker.io/kubesphere/k8s-dns-node-cache:1.23.1 + - docker.io/openebs/provisioner-localpv:3.5.0 + - docker.io/kubesphere/coredns:v1.11.1 + - docker.io/kubesphere/kube-apiserver:{{ .kube_version }} + - docker.io/kubesphere/kube-controller-manager:{{ .kube_version }} + - docker.io/kubesphere/kube-proxy:{{ .kube_version }} + - docker.io/kubesphere/kube-scheduler:{{ .kube_version }} + - docker.io/kubesphere/pause:3.8 + - quay.io/tigera/operator:v1.34.5 + - docker.io/calico/ctl:v3.28.2 + - docker.io/calico/typha:v3.28.2 + - docker.io/calico/apiserver:v3.28.2 + - docker.io/calico/kube-controllers:v3.28.2 + - docker.io/calico/node:v3.28.2 + - docker.io/calico/pod2daemon-flexvol:v3.28.2 + - docker.io/calico/cni:v3.28.2 + - docker.io/calico/node-driver-registrar:v3.28.2 + - docker.io/calico/csi:v3.28.2 diff --git a/builtin/core/defaults/config/v1.30.yaml b/builtin/core/defaults/config/v1.30.yaml new file mode 100644 index 000000000..0adfe2d9d --- /dev/null +++ b/builtin/core/defaults/config/v1.30.yaml @@ -0,0 +1,104 @@ +apiVersion: kubekey.kubesphere.io/v1 +kind: Config +spec: + download: + # if set as "cn", so that online downloads will try to use available domestic sources whenever possible. + zone: "" + kubernetes: + kube_version: {{ .kube_version }} + # helm binary + helm_version: v3.13.3 + etcd: + # etcd binary + etcd_version: v3.5.10 + image_registry: + # ========== image registry ========== + # keepalived image tag. Used for load balancing when there are multiple image registry nodes. + keepalived_version: 2.0.20 + # ========== image registry: harbor ========== + # harbor image tag + harbor_version: v2.10.1 + # docker-compose binary + dockercompose_version: v2.20.3 + # ========== image registry: docker-registry ========== + # docker-registry image tag + docker_registry_version: 2.8.3 + cri: + # support: containerd,docker + container_manager: containerd + sandbox_image: + tag: "3.8" + # ========== cri ========== + # crictl binary + crictl_version: v1.30.0 + # ========== cri: docker ========== + # docker binary + docker_version: 24.0.7 + # cridockerd. Required when kube_version is greater than 1.24 + cridockerd_version: v0.3.1 + # ========== cri: containerd ========== + # containerd binary + containerd_version: v1.7.6 + # runc binary + runc_version: v1.1.7 + cni: + multus: + image: + tag: v4.2.1 + # ========== cni ========== + # cni_plugins binary (optional) + # cni_plugins_version: v1.2.0 + # ========== cni: calico ========== + # calicoctl binary + calico_version: v3.28.2 + # ========== cni: cilium ========== + # cilium helm + cilium_version: 1.15.4 + # ========== cni: kubeovn ========== + # kubeovn helm + kubeovn_version: 1.13.0 + # ========== cni: hybridnet ========== + # hybridnet helm + hybridnet_version: 0.6.8 + storage_class: + # ========== storageclass ========== + # ========== storageclass: local ========== + local: + provisioner_image: + tag: 4.0.0 + linux_utils_image: + tag: 4.0.0 + # ========== storageclass: nfs ========== + # nfs provisioner helm version + nfs_provisioner_version: 4.1.0 + dns: + dns_image: + tag: v1.11.1 + dns_cache_image: + tag: 1.23.1 + image_manifests: + - docker.io/calico/apiserver:v3.28.2 + - docker.io/calico/cni:v3.28.2 + - docker.io/calico/csi:v3.28.2 + - docker.io/calico/kube-controllers:v3.28.2 + - docker.io/calico/node-driver-registrar:v3.28.2 + - docker.io/calico/node:v3.28.2 + - docker.io/calico/pod2daemon-flexvol:v3.28.2 + - docker.io/kubesphere/k8s-dns-node-cache:1.23.1 + - docker.io/openebs/provisioner-localpv:4.0.0 + - docker.io/kubesphere/coredns:v1.11.1 + - docker.io/kubesphere/kube-apiserver:{{ .kube_version }} + - docker.io/kubesphere/kube-controller-manager:{{ .kube_version }} + - docker.io/kubesphere/kube-proxy:{{ .kube_version }} + - docker.io/kubesphere/kube-scheduler:{{ .kube_version }} + - docker.io/kubesphere/pause:3.8 + - quay.io/tigera/operator:v1.34.5 + - docker.io/calico/ctl:v3.28.2 + - docker.io/calico/typha:v3.28.2 + - docker.io/calico/apiserver:v3.28.2 + - docker.io/calico/kube-controllers:v3.28.2 + - docker.io/calico/node:v3.28.2 + - docker.io/calico/pod2daemon-flexvol:v3.28.2 + - docker.io/calico/cni:v3.28.2 + - docker.io/calico/node-driver-registrar:v3.28.2 + - docker.io/calico/csi:v3.28.2 diff --git a/builtin/core/defaults/config/v1.31.yaml b/builtin/core/defaults/config/v1.31.yaml new file mode 100644 index 000000000..c6abfcde4 --- /dev/null +++ b/builtin/core/defaults/config/v1.31.yaml @@ -0,0 +1,104 @@ +apiVersion: kubekey.kubesphere.io/v1 +kind: Config +spec: + download: + # if set as "cn", so that online downloads will try to use available domestic sources whenever possible. + zone: "" + kubernetes: + kube_version: {{ .kube_version }} + # helm binary + helm_version: v3.13.3 + etcd: + # etcd binary + etcd_version: v3.5.11 + image_registry: + # ========== image registry ========== + # keepalived image tag. Used for load balancing when there are multiple image registry nodes. + keepalived_version: 2.0.20 + # ========== image registry: harbor ========== + # harbor image tag + harbor_version: v2.10.1 + # docker-compose binary + dockercompose_version: v2.20.3 + # ========== image registry: docker-registry ========== + # docker-registry image tag + docker_registry_version: 2.8.3 + cri: + # support: containerd,docker + container_manager: containerd + sandbox_image: + tag: "3.8" + # ========== cri ========== + # crictl binary + crictl_version: v1.31.0 + # ========== cri: docker ========== + # docker binary + docker_version: 24.0.7 + # cridockerd. Required when kube_version is greater than 1.24 + cridockerd_version: v0.3.1 + # ========== cri: containerd ========== + # containerd binary + containerd_version: v1.7.6 + # runc binary + runc_version: v1.1.7 + cni: + multus: + image: + tag: v4.2.1 + # ========== cni ========== + # cni_plugins binary (optional) + # cni_plugins_version: v1.2.0 + # ========== cni: calico ========== + # calicoctl binary + calico_version: v3.28.2 + # ========== cni: cilium ========== + # cilium helm + cilium_version: 1.15.4 + # ========== cni: kubeovn ========== + # kubeovn helm + kubeovn_version: 1.13.0 + # ========== cni: hybridnet ========== + # hybridnet helm + hybridnet_version: 0.6.8 + storage_class: + # ========== storageclass ========== + # ========== storageclass: local ========== + local: + provisioner_image: + tag: 4.1.0 + linux_utils_image: + tag: 4.1.0 + # ========== storageclass: nfs ========== + # nfs provisioner helm version + nfs_provisioner_version: 4.2.0 + dns: + dns_image: + tag: v1.11.1 + dns_cache_image: + tag: 1.23.1 + image_manifests: + - docker.io/calico/apiserver:v3.28.2 + - docker.io/calico/cni:v3.28.2 + - docker.io/calico/csi:v3.28.2 + - docker.io/calico/kube-controllers:v3.28.2 + - docker.io/calico/node-driver-registrar:v3.28.2 + - docker.io/calico/node:v3.28.2 + - docker.io/calico/pod2daemon-flexvol:v3.28.2 + - docker.io/kubesphere/k8s-dns-node-cache:1.23.1 + - docker.io/openebs/provisioner-localpv:4.1.0 + - docker.io/kubesphere/coredns:v1.11.1 + - docker.io/kubesphere/kube-apiserver:{{ .kube_version }} + - docker.io/kubesphere/kube-controller-manager:{{ .kube_version }} + - docker.io/kubesphere/kube-proxy:{{ .kube_version }} + - docker.io/kubesphere/kube-scheduler:{{ .kube_version }} + - docker.io/kubesphere/pause:3.8 + - quay.io/tigera/operator:v1.34.5 + - docker.io/calico/ctl:v3.28.2 + - docker.io/calico/typha:v3.28.2 + - docker.io/calico/apiserver:v3.28.2 + - docker.io/calico/kube-controllers:v3.28.2 + - docker.io/calico/node:v3.28.2 + - docker.io/calico/pod2daemon-flexvol:v3.28.2 + - docker.io/calico/cni:v3.28.2 + - docker.io/calico/node-driver-registrar:v3.28.2 + - docker.io/calico/csi:v3.28.2 diff --git a/builtin/core/defaults/config/v1.32.yaml b/builtin/core/defaults/config/v1.32.yaml new file mode 100644 index 000000000..a8a9b9f18 --- /dev/null +++ b/builtin/core/defaults/config/v1.32.yaml @@ -0,0 +1,104 @@ +apiVersion: kubekey.kubesphere.io/v1 +kind: Config +spec: + download: + # if set as "cn", so that online downloads will try to use available domestic sources whenever possible. + zone: "" + kubernetes: + kube_version: {{ .kube_version }} + # helm binary + helm_version: v3.14.3 + etcd: + # etcd binary + etcd_version: v3.5.11 + image_registry: + # ========== image registry ========== + # keepalived image tag. Used for load balancing when there are multiple image registry nodes. + keepalived_version: 2.0.20 + # ========== image registry: harbor ========== + # harbor image tag + harbor_version: v2.10.1 + # docker-compose binary + dockercompose_version: v2.20.3 + # ========== image registry: docker-registry ========== + # docker-registry image tag + docker_registry_version: 2.8.3 + cri: + # support: containerd,docker + container_manager: containerd + sandbox_image: + tag: "3.8" + # ========== cri ========== + # crictl binary + crictl_version: v1.32.0 + # ========== cri: docker ========== + # docker binary + docker_version: 24.0.7 + # cridockerd. Required when kube_version is greater than 1.24 + cridockerd_version: v0.3.1 + # ========== cri: containerd ========== + # containerd binary + containerd_version: v1.7.6 + # runc binary + runc_version: v1.1.7 + cni: + multus: + image: + tag: v4.3.0 + # ========== cni ========== + # cni_plugins binary (optional) + # cni_plugins_version: v1.2.0 + # ========== cni: calico ========== + # calicoctl binary + calico_version: v3.28.2 + # ========== cni: cilium ========== + # cilium helm + cilium_version: 1.15.4 + # ========== cni: kubeovn ========== + # kubeovn helm + kubeovn_version: 1.13.0 + # ========== cni: hybridnet ========== + # hybridnet helm + hybridnet_version: 0.6.8 + storage_class: + # ========== storageclass ========== + # ========== storageclass: local ========== + local: + provisioner_image: + tag: 4.2.0 + linux_utils_image: + tag: 4.2.0 + # ========== storageclass: nfs ========== + # nfs provisioner helm version + nfs_provisioner_version: 4.2.0 + dns: + dns_image: + tag: v1.11.1 + dns_cache_image: + tag: 1.24.0 + image_manifests: + - docker.io/calico/apiserver:v3.28.2 + - docker.io/calico/cni:v3.28.2 + - docker.io/calico/csi:v3.28.2 + - docker.io/calico/kube-controllers:v3.28.2 + - docker.io/calico/node-driver-registrar:v3.28.2 + - docker.io/calico/node:v3.28.2 + - docker.io/calico/pod2daemon-flexvol:v3.28.2 + - docker.io/kubesphere/k8s-dns-node-cache:1.24.0 + - docker.io/openebs/provisioner-localpv:4.2.0 + - docker.io/kubesphere/coredns:v1.11.1 + - docker.io/kubesphere/kube-apiserver:{{ .kube_version }} + - docker.io/kubesphere/kube-controller-manager:{{ .kube_version }} + - docker.io/kubesphere/kube-proxy:{{ .kube_version }} + - docker.io/kubesphere/kube-scheduler:{{ .kube_version }} + - docker.io/kubesphere/pause:3.8 + - quay.io/tigera/operator:v1.34.5 + - docker.io/calico/ctl:v3.28.2 + - docker.io/calico/typha:v3.28.2 + - docker.io/calico/apiserver:v3.28.2 + - docker.io/calico/kube-controllers:v3.28.2 + - docker.io/calico/node:v3.28.2 + - docker.io/calico/pod2daemon-flexvol:v3.28.2 + - docker.io/calico/cni:v3.28.2 + - docker.io/calico/node-driver-registrar:v3.28.2 + - docker.io/calico/csi:v3.28.2 diff --git a/builtin/core/defaults/config/v1.33.yaml b/builtin/core/defaults/config/v1.33.yaml new file mode 100644 index 000000000..8aef1f25e --- /dev/null +++ b/builtin/core/defaults/config/v1.33.yaml @@ -0,0 +1,104 @@ +apiVersion: kubekey.kubesphere.io/v1 +kind: Config +spec: + download: + # if set as "cn", so that online downloads will try to use available domestic sources whenever possible. + zone: "" + kubernetes: + kube_version: {{ .kube_version }} + # helm binary + helm_version: v3.18.5 + etcd: + # etcd binary + etcd_version: v3.5.11 + image_registry: + # ========== image registry ========== + # keepalived image tag. Used for load balancing when there are multiple image registry nodes. + keepalived_version: 2.0.20 + # ========== image registry: harbor ========== + # harbor image tag + harbor_version: v2.10.1 + # docker-compose binary + dockercompose_version: v2.20.3 + # ========== image registry: docker-registry ========== + # docker-registry image tag + docker_registry_version: 2.8.3 + cri: + # support: containerd,docker + container_manager: containerd + sandbox_image: + tag: "3.9" + # ========== cri ========== + # crictl binary + crictl_version: v1.33.0 + # ========== cri: docker ========== + # docker binary + docker_version: 24.0.7 + # cridockerd. Required when kube_version is greater than 1.24 + cridockerd_version: v0.3.1 + # ========== cri: containerd ========== + # containerd binary + containerd_version: v1.7.6 + # runc binary + runc_version: v1.1.7 + cni: + multus: + image: + tag: v4.3.0 + # ========== cni ========== + # cni_plugins binary (optional) + # cni_plugins_version: v1.2.0 + # ========== cni: calico ========== + # calicoctl binary + calico_version: v3.28.2 + # ========== cni: cilium ========== + # cilium helm + cilium_version: 1.15.4 + # ========== cni: kubeovn ========== + # kubeovn helm + kubeovn_version: 1.13.0 + # ========== cni: hybridnet ========== + # hybridnet helm + hybridnet_version: 0.6.8 + storage_class: + # ========== storageclass ========== + # ========== storageclass: local ========== + local: + provisioner_image: + tag: 4.2.0 + linux_utils_image: + tag: 4.2.0 + # ========== storageclass: nfs ========== + # nfs provisioner helm version + nfs_provisioner_version: 4.3.0 + dns: + dns_image: + tag: v1.12.1 + dns_cache_image: + tag: 1.24.0 + image_manifests: + - docker.io/calico/apiserver:v3.28.2 + - docker.io/calico/cni:v3.28.2 + - docker.io/calico/csi:v3.28.2 + - docker.io/calico/kube-controllers:v3.28.2 + - docker.io/calico/node-driver-registrar:v3.28.2 + - docker.io/calico/node:v3.28.2 + - docker.io/calico/pod2daemon-flexvol:v3.28.2 + - docker.io/kubesphere/k8s-dns-node-cache:1.24.0 + - docker.io/openebs/provisioner-localpv:4.2.0 + - docker.io/kubesphere/coredns:v1.12.1 + - docker.io/kubesphere/kube-apiserver:{{ .kube_version }} + - docker.io/kubesphere/kube-controller-manager:{{ .kube_version }} + - docker.io/kubesphere/kube-proxy:{{ .kube_version }} + - docker.io/kubesphere/kube-scheduler:{{ .kube_version }} + - docker.io/kubesphere/pause:3.9 + - quay.io/tigera/operator:v1.34.5 + - docker.io/calico/ctl:v3.28.2 + - docker.io/calico/typha:v3.28.2 + - docker.io/calico/apiserver:v3.28.2 + - docker.io/calico/kube-controllers:v3.28.2 + - docker.io/calico/node:v3.28.2 + - docker.io/calico/pod2daemon-flexvol:v3.28.2 + - docker.io/calico/cni:v3.28.2 + - docker.io/calico/node-driver-registrar:v3.28.2 + - docker.io/calico/csi:v3.28.2 diff --git a/builtin/core/defaults/inventory/localhost.yaml b/builtin/core/defaults/inventory/localhost.yaml new file mode 100644 index 000000000..b7f723366 --- /dev/null +++ b/builtin/core/defaults/inventory/localhost.yaml @@ -0,0 +1,38 @@ +apiVersion: kubekey.kubesphere.io/v1 +kind: Inventory +metadata: + name: default +spec: + hosts: # your can set all nodes here. or set nodes on special groups. +# node1: +# connector: +# type: ssh +# host: node1 +# port: 22 +# user: root +# password: 123456 + groups: + # all kubernetes nodes. + k8s_cluster: + groups: + - kube_control_plane + - kube_worker + # control_plane nodes + kube_control_plane: + hosts: + - localhost + # worker nodes + kube_worker: + hosts: + - localhost + # etcd nodes when etcd_deployment_type is external + etcd: + hosts: + - localhost +# image_registry: +# hosts: +# - localhost + # nfs nodes for registry storage. and kubernetes nfs storage +# nfs: +# hosts: +# - localhost diff --git a/builtin/core/fs.go b/builtin/core/fs.go new file mode 100644 index 000000000..256dee0b5 --- /dev/null +++ b/builtin/core/fs.go @@ -0,0 +1,30 @@ +//go:build builtin +// +build builtin + +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +import ( + "embed" +) + +//go:embed playbooks roles +var BuiltinPlaybook embed.FS + +//go:embed defaults +var Defaults embed.FS diff --git a/builtin/core/playbooks/add_nodes.yaml b/builtin/core/playbooks/add_nodes.yaml new file mode 100644 index 000000000..0b9b8990a --- /dev/null +++ b/builtin/core/playbooks/add_nodes.yaml @@ -0,0 +1,52 @@ +--- +- import_playbook: hook/pre_install.yaml + +# Load default variables and perform prechecks on all hosts +- hosts: + - all + gather_facts: true + roles: + - defaults + - precheck + - native/root + +# Download all required software and generate certificates on the localhost +- hosts: + - localhost + gather_facts: true + roles: + - certs/init + - download + +# Initialize all nodes and install necessary software packages +- hosts: + - etcd + - k8s_cluster + - image_registry + - nfs + roles: + - native + +# Install the etcd cluster +- hosts: + - etcd + gather_facts: true + roles: + - etcd + +- hosts: + - k8s_cluster + roles: + - role: cri + when: or (.add_nodes | default list | empty) (.add_nodes | default list | has .inventory_hostname) + - role: kubernetes/init-kubernetes + when: or (.add_nodes | default list | empty) (.add_nodes | default list | has .inventory_hostname) + - role: kubernetes/pre-kubernetes + when: or (.add_nodes | default list | empty) (.add_nodes | default list | has .inventory_hostname) + - role: kubernetes/join-kubernetes + when: or (.add_nodes | default list | empty) (.add_nodes | default list | has .inventory_hostname) + - role: kubernetes/certs + when: + - or (.add_nodes | default list | empty) (.add_nodes | default list | has .inventory_hostname) + - .groups.kube_control_plane | default list | has .inventory_hostname + - .kubernetes.certs.renew diff --git a/builtin/core/playbooks/artifact_export.yaml b/builtin/core/playbooks/artifact_export.yaml new file mode 100644 index 000000000..849defff4 --- /dev/null +++ b/builtin/core/playbooks/artifact_export.yaml @@ -0,0 +1,18 @@ +--- +# Load default variables and perform prechecks on all hosts +- hosts: + - all + gather_facts: true + roles: + - defaults + - native/root + +# Download all required software and generate certificates on the localhost +- hosts: + - localhost + roles: + - download + tasks: + - name: Export artifact + command: | + cd {{ .binary_dir }} && tar -czvf {{ .artifact_file }} * diff --git a/builtin/core/playbooks/artifact_images.yaml b/builtin/core/playbooks/artifact_images.yaml new file mode 100644 index 000000000..de59659bf --- /dev/null +++ b/builtin/core/playbooks/artifact_images.yaml @@ -0,0 +1,67 @@ +--- +# Load default variables and perform prechecks on all hosts +- hosts: + - localhost + tags: ["always"] + roles: + - defaults + - native/root + +- hosts: + - localhost + tasks: + - name: PullImage | Download container images + tags: ["pull","image_registry"] + image: + pull: + auths: "{{ .cri.registry.auths | toJson }}" + images_dir: >- + {{ .binary_dir }}/images/ + manifests: "{{ .image_manifests | toJson }}" + when: + - .image_manifests | default list | empty | not + - .download.download_image + - name: PushImage | Push images to registry + tags: ["push","image_registry"] + block: + - name: PushImage | Ensure Harbor project exists for each image + when: .image_registry.type | eq "harbor" + command: | + # Traverse first-level subdirectories in images_dir, skipping 'blobs' + + for registry_dir in {{ .binary_dir }}/images/*; do + if [ ! -d "$registry_dir" ] || [ "$(basename "$registry_dir")" = "blobs" ]; then + continue + fi + + # Traverse second-level subdirectories in each registry_dir + for project_dir in "$registry_dir"/*; do + if [ ! -d "$project_dir" ]; then + continue + fi + + project=$(basename "$project_dir") + + # Check if the Harbor project exists; create it if it does not + resp=$(curl -u "{{ .image_registry.auth.username }}:{{ .image_registry.auth.password }}" -k -X GET "https://{{ .image_registry.auth.registry }}/api/v2.0/projects/${project}") + if echo "$resp" | grep -q '"code":"NOT_FOUND"'; then + curl -u "{{ .image_registry.auth.username }}:{{ .image_registry.auth.password }}" -k -X POST \ + -H "Content-Type: application/json" \ + "https://{{ .image_registry.auth.registry }}/api/v2.0/projects" \ + -d "{ \"project_name\": \"${project}\", \"public\": true}" + fi + done + done + - name: PushImage | Push images package to image registry + image: + push: + images_dir: >- + {{ .binary_dir }}/images/ + dest: >- + {{ .image_registry.auth.registry }}/{{ .module.image.src.reference.repository }}:{{ .module.image.src.reference.reference }} + username: >- + {{ .image_registry.auth.username }} + password: >- + {{ .image_registry.auth.password }} + skip_tls_verify: true + diff --git a/builtin/core/playbooks/certs_renew.yaml b/builtin/core/playbooks/certs_renew.yaml new file mode 100644 index 000000000..8068e98d9 --- /dev/null +++ b/builtin/core/playbooks/certs_renew.yaml @@ -0,0 +1,18 @@ +--- +# Load default variables and perform prechecks on all hosts +- hosts: + - all + gather_facts: true + roles: + - defaults + - native/root + +- hosts: + - localhost + roles: + - cert/init + +- hosts: + - all + roles: + - role: certs/renew diff --git a/builtin/core/playbooks/create_cluster.yaml b/builtin/core/playbooks/create_cluster.yaml new file mode 100644 index 000000000..d2aadf5cb --- /dev/null +++ b/builtin/core/playbooks/create_cluster.yaml @@ -0,0 +1,72 @@ +--- +- import_playbook: hook/pre_install.yaml + +# Load default variables and perform prechecks on all hosts +- hosts: + - all + gather_facts: true + roles: + - defaults + - precheck + - native/root + +# Download all required software and generate certificates on the localhost +- hosts: + - localhost + roles: + - certs/init + - download + +# Initialize all nodes and install necessary software packages +- hosts: + - etcd + - k8s_cluster + - image_registry + - nfs + roles: + - native + +# Install the etcd cluster +- hosts: + - etcd + roles: + - etcd + +# Install the private image registry +- hosts: + - image_registry + roles: + - image-registry + +# Install the Kubernetes cluster +- hosts: + - k8s_cluster + gather_facts: true + roles: + - cri + - kubernetes/pre-kubernetes + - kubernetes/init-kubernetes + - role: kubernetes/join-kubernetes + when: + - .init_kubernetes_node | ne .inventory_hostname + - .kubernetes_install_LoadState.stdout | eq "not-found" + - role: kubernetes/certs + when: + - .kubernetes.certs.renew + - .groups.kube_control_plane | default list | has .inventory_hostname + post_tasks: + - name: Add custom labels to the cluster nodes + command: | + {{- range $k, $v := .kubernetes.custom_labels }} + /usr/local/bin/kubectl label --overwrite node {{ $.hostname }} {{ $k }}={{ $v }} + {{- end }} + when: .kubernetes.custom_label | empty | not + +# Install Kubernetes cluster software components (CNI and storage class) on a random control plane node +- hosts: + - kube_control_plane|random + roles: + - cni + - storage-class + +- import_playbook: hook/post_install.yaml \ No newline at end of file diff --git a/builtin/core/playbooks/delete_cluster.yaml b/builtin/core/playbooks/delete_cluster.yaml new file mode 100644 index 000000000..f0b6ba696 --- /dev/null +++ b/builtin/core/playbooks/delete_cluster.yaml @@ -0,0 +1,39 @@ +--- +# Load default variables and perform prechecks on all hosts +- hosts: + - all + gather_facts: true + roles: + - defaults + - native/root + +- hosts: + - k8s_cluster + roles: + - uninstall/kubernetes + - role: uninstall/cri + when: + - .deleteCRI + - .groups.image_registry | default list | has .inventory_hostname | not + post_tasks: + - name: delete localDNS file + ignore_errors: true + loop: "{{ .native.localDNS | toJson }}" + command: | + sed -i ':a;$!{N;ba};s@# kubekey hosts BEGIN.*# kubekey hosts END@@' {{ .item }} + sed -i ':a;$!{N;ba};s@# kubekey kubernetes control_plane_endpoint BEGIN.*# kubekey kubernetes control_plane_endpoint END@@' {{ .item }} + sed -i ':a;$!{N;ba};s@# kubekey image_registry control_plane_endpoint BEGIN.*# kubekey image_registry control_plane_endpoint END@@' {{ .item }} + when: .deleteDNS + +- hosts: + - etcd + roles: + - role: uninstall/etcd + when: .deleteETCD + +- hosts: + - image_registry + roles: + - role: uninstall/image-registry + when: + - .deleteImageRegistry diff --git a/builtin/core/playbooks/delete_nodes.yaml b/builtin/core/playbooks/delete_nodes.yaml new file mode 100644 index 000000000..adf46b8eb --- /dev/null +++ b/builtin/core/playbooks/delete_nodes.yaml @@ -0,0 +1,81 @@ +--- +# Load default variables and perform prechecks on all hosts +- hosts: + - all + gather_facts: true + roles: + - defaults + - precheck + - native/root + +- hosts: + - kube_control_plane + gather_facts: true + tasks: + - name: DeleteNode | Ensure at least one control plane node remains in the cluster + run_once: true + command: | + {{- $cpNodes := list -}} + {{- range .groups.kube_control_plane -}} + {{- if index $.hostvars . "kubernetes_install_LoadState" "stdout" | eq "loaded" -}} + {{- $cpNodes = append $cpNodes . -}} + {{- end -}} + {{- end -}} + {{- if (subtractList $cpNodes .delete_nodes) | empty }} + echo "At least one control plane node must be retained in the cluster." >&2 + exit 1 + {{- end }} + +- hosts: + - k8s_cluster + pre_tasks: + - name: DeleteNode | Remove node from Kubernetes cluster + when: .delete_nodes | default list | has .inventory_hostname + command: | + if kubectl get node {{ .hostname }} > /dev/null 2>&1; then + kubectl cordon {{ .hostname }} + if [ $(kubectl get nodes --no-headers | wc -l) -gt 1 ]; then + kubectl drain {{ .hostname }} --ignore-daemonsets --delete-emptydir-data --force --disable-eviction + else + kubectl drain {{ .hostname }} --ignore-daemonsets --delete-emptydir-data --force + fi + {{- if .cni.type | eq "calico" }} + calicoctl delete node {{ .hostname }} + {{- end }} + kubectl delete node {{ .hostname }} + fi + roles: + - role: uninstall/kubernetes + when: .delete_nodes | default list | has .inventory_hostname + - role: uninstall/cri + when: + - .deleteCRI + - .groups.image_registry | default list | has .inventory_hostname | not + - .delete_nodes | default list | has .inventory_hostname + post_tasks: + - name: DeleteNode | Clean up local DNS configuration files + ignore_errors: true + loop: "{{ .native.localDNS | toJson }}" + command: | + sed -i ':a;$!{N;ba};s@# kubekey hosts BEGIN.*# kubekey hosts END@@' {{ .item }} + sed -i ':a;$!{N;ba};s@# kubekey kubernetes control_plane_endpoint BEGIN.*# kubekey kubernetes control_plane_endpoint END@@' {{ .item }} + sed -i ':a;$!{N;ba};s@# kubekey image_registry control_plane_endpoint BEGIN.*# kubekey image_registry control_plane_endpoint END@@' {{ .item }} + when: + - .deleteDNS + - .delete_nodes | default list | has .inventory_hostname + +- hosts: + - etcd + roles: + - role: uninstall/etcd + when: + - .deleteETCD + - .delete_nodes | default list | has .inventory_hostname + +- hosts: + - image_registry + roles: + - role: uninstall/image-registry + when: + - .deleteImageRegistry + - .delete_nodes | default list | has .inventory_hostname diff --git a/builtin/core/playbooks/delete_registry.yaml b/builtin/core/playbooks/delete_registry.yaml new file mode 100644 index 000000000..c54a7a495 --- /dev/null +++ b/builtin/core/playbooks/delete_registry.yaml @@ -0,0 +1,13 @@ +--- +# Load default variables and perform prechecks on all hosts +- hosts: + - all + gather_facts: true + roles: + - defaults + - native/root + +- hosts: + - image_registry + roles: + - role: uninstall/image-registry \ No newline at end of file diff --git a/builtin/core/playbooks/hook/post_install.yaml b/builtin/core/playbooks/hook/post_install.yaml new file mode 100644 index 000000000..2eed64bd8 --- /dev/null +++ b/builtin/core/playbooks/hook/post_install.yaml @@ -0,0 +1,32 @@ +--- +- name: Post | Apply Security Enhancements + hosts: + - all + roles: + - role: security + when: .security_enhancement + +- name: Post | Run Post-Installation Scripts + hosts: + - all + tasks: + - name: Post | Copy post-installation scripts to remote hosts + ignore_errors: true + copy: + src: >- + {{ .scripts_dir }}/post_install_{{ .inventory_hostname }}.sh + dest: >- + /etc/kubekey/scripts/post_install_{{ .inventory_hostname }}.sh + mode: 0755 + register: post_install_copy_result + + - name: Post | Execute post-installation scripts on remote hosts + when: .post_install_copy_result.error | empty + command: | + for file in /etc/kubekey/scripts/post_install_*.sh; do + if [ -f "$file" ]; then + # Make the script executable and run it + chmod +x "$file" + "$file" + fi + done \ No newline at end of file diff --git a/builtin/core/playbooks/hook/pre_install.yaml b/builtin/core/playbooks/hook/pre_install.yaml new file mode 100644 index 000000000..251673163 --- /dev/null +++ b/builtin/core/playbooks/hook/pre_install.yaml @@ -0,0 +1,25 @@ +--- +- name: Pre | Run Pre-Installation Scripts + hosts: + - all + tasks: + - name: Pre | Copy pre-installation scripts to remote hosts + ignore_errors: true + copy: + src: >- + {{ .work_dir }}/scripts/pre_install_{{ .inventory_hostname }}.sh + dest: >- + /etc/kubekey/scripts/pre_install_{{ .inventory_hostname }}.sh + mode: 0755 + register: pre_install_copy_result + + - name: Pre | Execute pre-installation scripts on remote hosts + when: .pre_install_copy_result.error | empty + command: | + for file in /etc/kubekey/scripts/pre_install_*.sh; do + if [ -f "$file" ]; then + # Make the script executable and run it + chmod +x "$file" + "$file" + fi + done diff --git a/builtin/core/playbooks/init_os.yaml b/builtin/core/playbooks/init_os.yaml new file mode 100644 index 000000000..3f95feaf4 --- /dev/null +++ b/builtin/core/playbooks/init_os.yaml @@ -0,0 +1,26 @@ +--- +# Load default variables and perform prechecks on all hosts +- hosts: + - all + gather_facts: true + roles: + - defaults + - native/root + +# Download all required software and generate certificates on the localhost +- hosts: + - localhost + gather_facts: true + roles: + - certs/init + - download + +# Initialize all nodes and install necessary software packages +- hosts: + - etcd + - k8s_cluster + - image_registry + - nfs + roles: + - native + diff --git a/builtin/core/playbooks/init_registry.yaml b/builtin/core/playbooks/init_registry.yaml new file mode 100644 index 000000000..ada86a0f8 --- /dev/null +++ b/builtin/core/playbooks/init_registry.yaml @@ -0,0 +1,26 @@ +--- +# Load default variables and perform prechecks on all hosts +- hosts: + - all + tags: ["always"] + gather_facts: true + roles: + - defaults + - native/root + + +# Download all required software and generate certificates on the localhost +- hosts: + - localhost + gather_facts: true + roles: + - role: certs/init + - role: download + +# Initialize all nodes and install necessary software packages +- hosts: + - image_registry + tags: ["always"] + roles: + - role: native + - role: image-registry diff --git a/builtin/core/playbooks/precheck.yaml b/builtin/core/playbooks/precheck.yaml new file mode 100644 index 000000000..4e7e5f286 --- /dev/null +++ b/builtin/core/playbooks/precheck.yaml @@ -0,0 +1,9 @@ +--- +# Load default variables and perform prechecks on all hosts +- hosts: + - all + gather_facts: true + roles: + - defaults + - precheck + - native/root diff --git a/builtin/core/roles/certs/init/tasks/main.yaml b/builtin/core/roles/certs/init/tasks/main.yaml new file mode 100644 index 000000000..ab7b02a1b --- /dev/null +++ b/builtin/core/roles/certs/init/tasks/main.yaml @@ -0,0 +1,112 @@ +--- +- name: Cert | Generate the root CA certificate file + tags: ["always"] + gen_cert: + cn: root + date: "{{ .certs.ca.date }}" + policy: "{{ .certs.ca.gen_cert_policy }}" + out_key: >- + {{ .binary_dir }}/pki/root.key + out_cert: >- + {{ .binary_dir }}/pki/root.crt + +- name: Cert | Generate Kubernetes CA certificates + tags: ["kubernetes"] + block: + - name: Cert | Generate the Kubernetes CA certificate file + gen_cert: + root_key: >- + {{ .binary_dir }}/pki/root.key + root_cert: >- + {{ .binary_dir }}/pki/root.crt + cn: kubernetes-ca + is_ca: true + date: "{{ .certs.kubernetes_ca.date }}" + policy: "{{ .certs.kubernetes_ca.gen_cert_policy }}" + out_key: >- + {{ .binary_dir }}/pki/kubernetes.key + out_cert: >- + {{ .binary_dir }}/pki/kubernetes.crt + - name: Cert | Generate the front-proxy CA certificate for Kubernetes + gen_cert: + root_key: >- + {{ .binary_dir }}/pki/root.key + root_cert: >- + {{ .binary_dir }}/pki/root.crt + cn: front-proxy-ca + is_ca: true + date: "{{ .certs.front_proxy_ca.date }}" + policy: "{{ .certs.front_proxy_ca.gen_cert_policy }}" + out_key: >- + {{ .binary_dir }}/pki/front-proxy.key + out_cert: >- + {{ .binary_dir }}/pki/front-proxy.crt + +- name: Cert | Generate the etcd certificate file + tags: ["etcd"] + gen_cert: + root_key: >- + {{ .binary_dir }}/pki/root.key + root_cert: >- + {{ .binary_dir }}/pki/root.crt + cn: etcd + sans: >- + {{- $ips := list -}} + {{- range .groups.etcd | default list -}} + {{- $internalIPv4 := index $.hostvars . "internal_ipv4" | default "" -}} + {{- $internalIPv6 := index $.hostvars . "internal_ipv6" | default "" -}} + {{- if $internalIPv4 | empty | not -}} + {{- $ips = append $ips $internalIPv4 -}} + {{- end -}} + {{- if $internalIPv6 | empty | not -}} + {{- $ips = append $ips $internalIPv6 -}} + {{- end -}} + {{- end -}} + {{ $ips | toJson }} + date: "{{ .certs.etcd.date }}" + policy: "{{ .certs.etcd.gen_cert_policy }}" + out_key: >- + {{ .binary_dir }}/pki/etcd.key + out_cert: >- + {{ .binary_dir }}/pki/etcd.crt + when: .groups.etcd | default list | empty | not + +- name: Cert | Generate the image registry certificate file + tags: ["image_registry"] + gen_cert: + root_key: >- + {{ .binary_dir }}/pki/root.key + root_cert: >- + {{ .binary_dir }}/pki/root.crt + cn: image_registry + sans: >- + {{- $ips := list -}} + {{- if .image_registry.ha_vip | empty | not -}} + {{- $ips = append $ips .image_registry.ha_vip -}} + {{- end -}} + {{- range .groups.image_registry | default list -}} + {{- $internalIPv4 := index $.hostvars . "internal_ipv4" | default "" -}} + {{- $internalIPv6 := index $.hostvars . "internal_ipv6" | default "" -}} + {{- if $internalIPv4 | empty | not -}} + {{- $ips = append $ips $internalIPv4 -}} + {{- end -}} + {{- if $internalIPv6 | empty | not -}} + {{- $ips = append $ips $internalIPv6 -}} + {{- end -}} + {{- end -}} + {{ $ips | toJson }} + date: "{{ .certs.image_registry.date }}" + policy: "{{ .certs.image_registry.gen_cert_policy }}" + out_key: >- + {{ .binary_dir }}/pki/image_registry.key + out_cert: >- + {{ .binary_dir }}/pki/image_registry.crt + when: .groups.image_registry | default list | empty | not + +- name: Cert | Set ownership of the PKI directory to the sudo user + tags: ["kubernetes"] + block: + - name: Cert | Change ownership of the PKI directory to the sudo user + ignore_errors: true + command: | + chown -R ${SUDO_UID}:${SUDO_GID} {{ .binary_dir }}/pki diff --git a/builtin/core/roles/certs/renew/etcd/tasks/main.yaml b/builtin/core/roles/certs/renew/etcd/tasks/main.yaml new file mode 100644 index 000000000..838e12b55 --- /dev/null +++ b/builtin/core/roles/certs/renew/etcd/tasks/main.yaml @@ -0,0 +1,21 @@ +--- +- name: ETCD | Copy CA certificate to remote host + copy: + src: >- + {{ ..etcd.ca_file }} + dest: /etc/ssl/etcd/ssl/ca.crt + +- name: ETCD | Copy server certificate to remote host + copy: + src: >- + {{ .etcd.cert_file }} + dest: /etc/ssl/etcd/ssl/server.crt + +- name: ETCD | Copy server private key to remote host + copy: + src: >- + {{ .etcd.key_file }} + dest: /etc/ssl/etcd/ssl/server.key + +- name: ETCD | Restart etcd service to apply new certificates + command: systemctl restart etcd diff --git a/builtin/core/roles/certs/renew/image-registry/tasks/harbor.yaml b/builtin/core/roles/certs/renew/image-registry/tasks/harbor.yaml new file mode 100644 index 000000000..daed5f7f6 --- /dev/null +++ b/builtin/core/roles/certs/renew/image-registry/tasks/harbor.yaml @@ -0,0 +1,17 @@ +--- +- name: Harbor | Copy image registry certificate to remote host + copy: + src: >- + {{ .binary_dir }}/pki/image_registry.crt + dest: >- + /opt/harbor/{{ .image_registry.harbor_version }}/ssl/server.crt + +- name: Harbor | Copy image registry private key to remote host + copy: + src: >- + {{ .binary_dir }}/pki/image_registry.key + dest: >- + /opt/harbor/{{ .image_registry.harbor_version }}/ssl/server.key + +- name: Harbor | Restart Harbor service to apply new certificates + command: systemctl restart harbor.service diff --git a/builtin/core/roles/certs/renew/image-registry/tasks/main.yaml b/builtin/core/roles/certs/renew/image-registry/tasks/main.yaml new file mode 100644 index 000000000..8154f1df0 --- /dev/null +++ b/builtin/core/roles/certs/renew/image-registry/tasks/main.yaml @@ -0,0 +1,4 @@ +- include_tasks: harbor.yaml + when: .image_registry.type | eq "harbor" +- include_tasks: registry.yaml + when: .image_registry.type | eq "docker-registry" diff --git a/builtin/core/roles/certs/renew/image-registry/tasks/registry.yaml b/builtin/core/roles/certs/renew/image-registry/tasks/registry.yaml new file mode 100644 index 000000000..356c09805 --- /dev/null +++ b/builtin/core/roles/certs/renew/image-registry/tasks/registry.yaml @@ -0,0 +1,17 @@ +--- +- name: Docker Registry | Copy image registry certificate to remote host + copy: + src: >- + {{ .binary_dir }}/pki/image_registry.crt + dest: >- + /opt/docker-registry/{{ .image_registry.docker_registry_version }}/ssl/server.crt + +- name: Docker Registry | Copy image registry private key to remote host + copy: + src: >- + {{ .binary_dir }}/pki/image_registry.key + dest: >- + /opt/docker-registry/{{ .image_registry.docker_registry_version }}/ssl/server.key + +- name: Docker Registry | Restart registry service to apply new certificates + command: systemctl restart docker-registry.service diff --git a/builtin/core/roles/certs/renew/kubernetes/tasks/etcd.yaml b/builtin/core/roles/certs/renew/kubernetes/tasks/etcd.yaml new file mode 100644 index 000000000..dfd71d810 --- /dev/null +++ b/builtin/core/roles/certs/renew/kubernetes/tasks/etcd.yaml @@ -0,0 +1,21 @@ +--- +- name: ETCD | Copy CA certificate to remote host + copy: + src: >- + {{ .etcd.ca_file }} + dest: /etc/kubernetes/pki/etcd/ca.crt + mode: 0755 + +- name: ETCD | Copy client certificate to remote host + copy: + src: >- + {{ .etcd.cert_file }} + dest: /etc/kubernetes/pki/etcd/client.crt + mode: 0755 + +- name: ETCD | Copy client key to remote host + copy: + src: >- + {{ .etcd.key_file }} + dest: /etc/kubernetes/pki/etcd/client.key + mode: 0755 diff --git a/builtin/core/roles/certs/renew/kubernetes/tasks/kube.yaml b/builtin/core/roles/certs/renew/kubernetes/tasks/kube.yaml new file mode 100644 index 000000000..5e3ca3210 --- /dev/null +++ b/builtin/core/roles/certs/renew/kubernetes/tasks/kube.yaml @@ -0,0 +1,49 @@ +--- +- name: Kubernetes | Determine installed kubeadm version + run_once: true + command: kubeadm version -o short + register: kubeadm_install_version + +- name: Kubernetes | Renew certificates using kubeadm + run_once: true + command: | + {{- if .kubeadm_install_version.stdout | semverCompare "- + {{ .binary_dir }}/kubeconfig + +- name: Kubernetes | Distribute kubeconfig to remote host + copy: + src: >- + {{ .binary_dir }}/kubeconfig + dest: /root/.kube/config diff --git a/builtin/core/roles/certs/renew/kubernetes/tasks/main.yaml b/builtin/core/roles/certs/renew/kubernetes/tasks/main.yaml new file mode 100644 index 000000000..f65d382d6 --- /dev/null +++ b/builtin/core/roles/certs/renew/kubernetes/tasks/main.yaml @@ -0,0 +1,29 @@ +--- +- include_tasks: kube.yaml + +- include_tasks: etcd.yaml + when: + - .etcd.deployment_type | eq "external" + - .groups.etcd | default list | empty | not + +- name: Kubernetes | Restart Kubernetes control plane pods + command: | + {{- if .cri.container_manager | eq "docker" }} + # Restarting Kubernetes control plane pods using Docker + docker ps -af name=k8s_PODS_kube-apiserver* -q | xargs --no-run-if-empty docker rm -f + docker ps -af name=k8s_PODS_kube-controller-manager* -q | xargs --no-run-if-empty docker rm -f + docker ps -af name=k8s_PODS_kube-scheduler* -q | xargs --no-run-if-empty docker rm -f + {{- if .etcd.deployment_type | eq "docker" }} + # Restarting etcd pods managed by Docker + docker ps -af name=k8s_PODS_etcd* -q | xargs --no-run-if-empty docker rm -f + {{- end }} + {{- else }} + # Restarting Kubernetes control plane pods using crictl + crictl pods --name kube-apiserver-* -q | xargs -I% --no-run-if-empty bash -c 'crictl stopp % && crictl rmp %' + crictl pods --name kube-controller-manager-* -q | xargs -I% --no-run-if-empty bash -c 'crictl stopp % && crictl rmp %' + crictl pods --name kube-scheduler-* -q | xargs -I% --no-run-if-empty bash -c 'crictl stopp % && crictl rmp %' + {{- if .etcd.deployment_type | eq "internal" }} + # Restarting etcd pods managed by the container runtime + crictl pods --name etcd-* -q | xargs -I% --no-run-if-empty bash -c 'crictl stopp % && crictl rmp %' + {{- end }} + {{- end }} diff --git a/builtin/core/roles/certs/renew/meta/main.yaml b/builtin/core/roles/certs/renew/meta/main.yaml new file mode 100644 index 000000000..477da2a8c --- /dev/null +++ b/builtin/core/roles/certs/renew/meta/main.yaml @@ -0,0 +1,13 @@ +--- +dependencies: + - role: renew/etcd + tags: ["certs", "etcd"] + when: .groups.etcd | default list | has .inventory_hostname + + - role: renew/kubernetes + tags: ["certs", "kubernetes"] + when: .groups.kube_control_plane | default list | has .inventory_hostname + + - role: renew/image-registry + tags: ["certs", "image-registry"] + when: .groups.image_registry | default list | has .inventory_hostname diff --git a/builtin/core/roles/cni/calico/defaults/main.yaml b/builtin/core/roles/cni/calico/defaults/main.yaml new file mode 100644 index 000000000..69dbecaea --- /dev/null +++ b/builtin/core/roles/cni/calico/defaults/main.yaml @@ -0,0 +1,12 @@ +cni: + calico: + values: | + # calico helm values + tigeraOperator: + registry: {{ .image_registry.quayio_registry }} + calicoctl: + image: {{ .image_registry.dockerio_registry }}/calico/ctl + installation: + registry: {{ .image_registry.dockerio_registry }} + calicoNetwork: + bgp: Enabled \ No newline at end of file diff --git a/builtin/core/roles/cni/calico/tasks/main.yaml b/builtin/core/roles/cni/calico/tasks/main.yaml new file mode 100644 index 000000000..28644013e --- /dev/null +++ b/builtin/core/roles/cni/calico/tasks/main.yaml @@ -0,0 +1,33 @@ +--- +- name: Calico | Check if calicoctl is installed + ignore_errors: true + command: calicoctl version + register: calicoctl_install_version + register_type: yaml + +- name: Calico | Install calicoctl if it is not present + when: .calicoctl_install_version.error | empty | not + block: + - name: Calico | Copy calicoctl binary to remote node + copy: + src: >- + {{ .binary_dir }}/cni/calico/{{ .cni.calico_version }}/{{ .binary_type }}/calicoctl + dest: /usr/local/bin/calicoctl + mode: 0755 + +- name: Calico | Copy Calico Helm package to remote node + copy: + src: >- + {{ .binary_dir }}/cni/calico/tigera-operator-{{ .cni.calico_version }}.tgz + dest: >- + /etc/kubernetes/cni/tigera-operator-{{ .cni.calico_version }}.tgz + +- name: Calico | Generate custom values file for Calico + copy: + content: | + {{ .cni.calico.values }} + dest: /etc/kubernetes/cni/calico-values.yaml + +- name: Calico | Deploy Calico using Helm + command: | + helm upgrade --install --create-namespace --namespace tigera-operator calico /etc/kubernetes/cni/tigera-operator-{{ .cni.calico_version }}.tgz -f /etc/kubernetes/cni/calico-values.yaml diff --git a/builtin/core/roles/cni/cilium/defaults/main.yaml b/builtin/core/roles/cni/cilium/defaults/main.yaml new file mode 100644 index 000000000..0c9ad8c42 --- /dev/null +++ b/builtin/core/roles/cni/cilium/defaults/main.yaml @@ -0,0 +1,70 @@ +cni: + cilium: + values: | + # cilium helm values + image: + repository: {{ .image_registry.quayio_registry }}/cilium/cilium-cli + certgen: + image: + repository: {{ .image_registry.quayio_registry }}/cilium/certgen + hubble: + relay: + image: + repository: {{ .image_registry.quayio_registry }}/cilium/hubble-relay-ci + ui: + backend: + image: + repository: {{ .image_registry.quayio_registry }}/cilium/hubble-ui-backend + frontend: + image: + repository: {{ .image_registry.quayio_registry }}/cilium/hubble-ui + envoy: + image: + repository: {{ .image_registry.quayio_registry }}/cilium/cilium-envoy + operator: + replicas: 2 + image: + repository: {{ .image_registry.quayio_registry }}/cilium/operator + nodeinit: + image: + repository: {{ .image_registry.quayio_registry }}/cilium/startup-script + preflight: + image: + repository: {{ .image_registry.quayio_registry }}/cilium/cilium-ci + clustermesh: + apiserver: + image: + repository: {{ .image_registry.quayio_registry }}/cilium/clustermesh-apiserver-ci + authentication: + mutual: + spire: + install: + initImage: + repository: {{ .image_registry.dockerio_registry }}/library/busybox + agent: + image: + repository: {{ .image_registry.ghcrio_registry }}/spiffe/spire-agent + server: + image: + repository: {{ .ghcrio_registry }}/spiffe/spire-server + ipv4: + enabled: {{ .cni.ipv4_support }} + ipv6: + enabled: {{ .cni.ipv6_support }} + ipam: + operator: + {{- if .cni.ipv4_support }} + clusterPoolIPv4PodCIDRList: + - {{ .cni.ipv4_pods_cidr }} + clusterPoolIPv4MaskSize: {{ .cni.ipv4_mask_size }} + {{- end }} + {{- if .cni.ipv6_support }} + clusterPoolIPv6PodCIDRList: + - {{ .cni.ipv6_pods_cidr }} + clusterPoolIPv6MaskSize: {{ .cni.ipv6_mask_size }} + {{- end }} + {{- if not (.kubernetes.kube_proxy.enabled | default true) }} + kubeProxyReplacement: "true" + k8sServiceHost: {{ .kubernetes.control_plane_endpoint.host }} + k8sServicePort: {{ .kubernetes.control_plane_endpoint.port }} + {{- end }} \ No newline at end of file diff --git a/builtin/core/roles/cni/cilium/tasks/main.yaml b/builtin/core/roles/cni/cilium/tasks/main.yaml new file mode 100644 index 000000000..3b1aca893 --- /dev/null +++ b/builtin/core/roles/cni/cilium/tasks/main.yaml @@ -0,0 +1,18 @@ +--- +- name: Cilium | Ensure the cilium Helm chart archive is available + copy: + src: >- + {{ .binary_dir }}/cni/cilium/cilium-{{ .cni.cilium_version }}.tgz + dest: >- + /etc/kubernetes/cni/cilium-{{ .cni.cilium_version }}.tgz + +- name: Cilium | Create the cilium Helm custom values file + copy: + content: | + {{ .cni.cilium.values }} + dest: /etc/kubernetes/cni/cilium-values.yaml + +# See: https://docs.cilium.io/en/stable/installation/k8s-install-helm/ +- name: Cilium | Deploy cilium with Helm + command: | + helm upgrade --install --namespace kube-system cilium /etc/kubernetes/cni/cilium-{{ .cni.cilium_version }}.tgz -f /etc/kubernetes/cni/cilium-values.yaml diff --git a/builtin/core/roles/cni/defaults/main.yaml b/builtin/core/roles/cni/defaults/main.yaml new file mode 100644 index 000000000..8feaec936 --- /dev/null +++ b/builtin/core/roles/cni/defaults/main.yaml @@ -0,0 +1,17 @@ +cni: + # In Kubernetes, Pod CIDR supports IPv4, IPv6, and dual-stack. Specify as: + # "Single-stack IPv4": pod_cidr in "ipv4" format + # "Single-stack IPv6": pod_cidr in "ipv6" format + # "Dual-stack": pod_cidr in "ipv4,ipv6" format + ipv4_support: >- + {{ eq (.cni.pod_cidr | splitList "," | first | ipFamily) "IPv4" }} + ipv4_pods_cidr: >- + {{- if eq (.cni.pod_cidr | splitList "," | first | ipFamily) "IPv4" -}} + {{ .cni.pod_cidr | splitList "," | first }} + {{- end -}} + ipv6_support: >- + {{- eq (.cni.pod_cidr | default "10.233.64.0/18" | splitList "," | last | ipFamily) "IPv6" }} + ipv6_pods_cidr: >- + {{- if eq (.cni.pod_cidr | default "10.233.64.0/18" | splitList "," | last | ipFamily) "IPv6" -}} + {{ .cni.pod_cidr | default "10.233.64.0/18" | splitList "," | last }} + {{- end -}} diff --git a/builtin/core/roles/cni/flannel/defaults/main.yaml b/builtin/core/roles/cni/flannel/defaults/main.yaml new file mode 100644 index 000000000..ef3207f64 --- /dev/null +++ b/builtin/core/roles/cni/flannel/defaults/main.yaml @@ -0,0 +1,14 @@ +cni: + flannel: + # https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md + values: | + # flannel helm values + podCidr: {{ .cni.ipv4_pod_cidr }} + podCidrv6: {{ .cni.ipv6_pod_cidr }} + flannel: + image: + repository: {{ .image_registry.dockerio_registry }}/flannel/flannel + image_cni: + repository: {{ .image_registry.dockerio_registry }}/flannel/flannel-cni-plugin + # support "vxlan" and "host-gw" + backend: vxlan \ No newline at end of file diff --git a/builtin/core/roles/cni/flannel/tasks/main.yaml b/builtin/core/roles/cni/flannel/tasks/main.yaml new file mode 100644 index 000000000..47254aa6a --- /dev/null +++ b/builtin/core/roles/cni/flannel/tasks/main.yaml @@ -0,0 +1,18 @@ +--- +# For more information, see: https://github.com/flannel-io/flannel/blob/master/Documentation/kubernetes.md + +- name: Flannel | Sync flannel package to remote node + copy: + src: >- + {{ .binary_dir }}/cni/flannel/flannel.tgz + dest: /etc/kubernetes/cni/flannel.tgz + +- name: Flannel | Generate flannel custom values file + copy: + content: | + {{ .cni.flannel.values }} + dest: /etc/kubernetes/cni/flannel-values.yaml + +- name: Flannel | Install flannel using Helm + command: | + helm upgrade --install --create-namespace --namespace kube-flannel flannel /etc/kubernetes/cni/flannel.tgz -f /etc/kubernetes/cni/flannel-values.yaml diff --git a/builtin/core/roles/cni/hybridnet/defaults/main.yaml b/builtin/core/roles/cni/hybridnet/defaults/main.yaml new file mode 100644 index 000000000..8befc9093 --- /dev/null +++ b/builtin/core/roles/cni/hybridnet/defaults/main.yaml @@ -0,0 +1,6 @@ +cni: + hybridnet: + values: | + # hybridnet helm values + images: + registryURL: {{ .image_registry.dockerio_registry }} \ No newline at end of file diff --git a/builtin/core/roles/cni/hybridnet/tasks/main.yaml b/builtin/core/roles/cni/hybridnet/tasks/main.yaml new file mode 100644 index 000000000..d6fca9804 --- /dev/null +++ b/builtin/core/roles/cni/hybridnet/tasks/main.yaml @@ -0,0 +1,18 @@ +--- +- name: Hybridnet | Synchronize Hybridnet Helm chart package to remote node + copy: + src: >- + {{ .binary_dir }}/cni/hybridnet-{{ .cni.hybridnet_version }}.tgz + dest: >- + /etc/kubernetes/cni/hybridnet-{{ .cni.hybridnet_version }}.tgz + +- name: Hybridnet | Generate Hybridnet custom values file + copy: + content: | + {{ .cni.hybridnet.values }} + dest: /etc/kubernetes/cni/hybridnet-values.yaml + +# Reference: https://artifacthub.io/packages/helm/hybridnet/hybridnet +- name: Hybridnet | Install Hybridnet using Helm + command: | + helm upgrade --install --namespace kube-system hybridnet /etc/kubernetes/cni/hybridnet-{{ .cni.hybridnet_version }}.tgz -f /etc/kubernetes/cni/hybridnet-values.yaml diff --git a/builtin/core/roles/cni/kubeovn/defaults/main.yaml b/builtin/core/roles/cni/kubeovn/defaults/main.yaml new file mode 100644 index 000000000..03db06a37 --- /dev/null +++ b/builtin/core/roles/cni/kubeovn/defaults/main.yaml @@ -0,0 +1,33 @@ +cni: + kubeovn: + values: | + # kube-ovn helm values + global: + registry: + address: {{ .image_registry.dockerio_registry }}/kubeovn + {{- $ips := list }} + {{- range .groups.kube_control_plane | default list }} + {{- $internalIPv4 := index $.hostvars . "internal_ipv4" | default "" }} + {{- $internalIPv6 := index $.hostvars . "internal_ipv6" | default "" }} + {{- if $internalIPv4| empty | not }} + {{- $ips = append $ips $internalIPv4 }} + {{- else if $internalIPv6 | empty | not }} + {{- $ips = append $ips $internalIPv6 }} + {{- end }} + {{- end }} + MASTER_NODES: {{ $ips | join "," }} + networking: + NET_STACK: {{ if and .cni.ipv4_support (not .cni.ipv6_support) }}ipv4{{ else if and .cni.ipv6_support (not .cni.ipv4_support) }}ipv6{{ else if and .cni.ipv4_support .cni.ipv6_support }}dual_stack{{ end }} + {{- if and .cni.ipv4_support (not .cni.ipv6_support) }} + ipv4: + POD_CIDR: {{ .cni.ipv4_pods_cidr }} + SVC_CIDR: {{ .cni.service_cidr }} + {{ else if and .cni.ipv6_support (not .cni.ipv4_support) }} + ipv6: + POD_CIDR: {{ .cni.ipv6_pods_cidr }} + SVC_CIDR: {{ .cni.service_cidr }} + {{ else if and .cni.ipv4_support .cni.ipv6_support }} + dual_stack: + POD_CIDR: {{ .cni.ipv4_pods_cidr }},{{ .cni.ipv6_pods_cidr }} + SVC_CIDR: {{ .cni.service_cidr }} + {{- end }} \ No newline at end of file diff --git a/builtin/core/roles/cni/kubeovn/tasks/main.yaml b/builtin/core/roles/cni/kubeovn/tasks/main.yaml new file mode 100644 index 000000000..d2fdab479 --- /dev/null +++ b/builtin/core/roles/cni/kubeovn/tasks/main.yaml @@ -0,0 +1,27 @@ +--- +- name: Kubeovn | Synchronize Kube-OVN Helm chart package to remote node + copy: + src: >- + {{ .binary_dir }}/cni/kubeovn/kubeovn-{{ .cni.kubeovn_version }}.tgz + dest: >- + /etc/kubernetes/cni/kubeovn-{{ .cni.kubeovn_version }}.tgz + +- name: Kubeovn | Generate Kube-OVN custom values file + copy: + content: | + {{ .cni.kubeovn.values }} + dest: /etc/kubernetes/cni/kubeovn-values.yaml + +- name: Kubeovn | Add Kube-OVN labels to nodes + command: | + kubectl label node -lbeta.kubernetes.io/os=linux kubernetes.io/os=linux --overwrite + kubectl label node -lnode-role.kubernetes.io/control-plane kube-ovn/role=master --overwrite + +- name: Kubeovn | Install Kube-OVN using Helm with custom values + command: | + helm upgrade --install --namespace kubeovn-system kubeovn /etc/kubernetes/cni/kubeovn-{{ .cni.kubeovn_version }}.tgz -f /etc/kubernetes/cni/kubeovn-values.yaml + +# Reference: https://kubeovn.github.io/docs/stable/start/one-step-install/#helm-chart +- name: Kubeovn | Install Kube-OVN using Helm + command: | + helm upgrade --install --namespace kubeovn-system kubeovn /etc/kubernetes/cni/kubeovn-{{ .cni.kubeovn_version }}.tgz diff --git a/builtin/core/roles/cni/meta/main.yaml b/builtin/core/roles/cni/meta/main.yaml new file mode 100644 index 000000000..494a6619b --- /dev/null +++ b/builtin/core/roles/cni/meta/main.yaml @@ -0,0 +1,19 @@ +--- +dependencies: + - role: cni/multus + when: .cni.multus.enabled + + - role: cni/calico + when: .cni.type | eq "calico" + + - role: cni/cilium + when: .cni.type | eq "cilium" + + - role: cni/flannel + when: .cni.type | eq "flannel" + + - role: cni/kubeovn + when: .cni.type | eq "kubeovn" + + - role: cni/hybridnet + when: .cni.type | eq "hybridnet" diff --git a/builtin/core/roles/cni/multus/tasks/main.yaml b/builtin/core/roles/cni/multus/tasks/main.yaml new file mode 100644 index 000000000..ef6b9d374 --- /dev/null +++ b/builtin/core/roles/cni/multus/tasks/main.yaml @@ -0,0 +1,9 @@ +--- +- name: Multus | Generate Multus configuration YAML file + template: + src: multus/multus.yaml + dest: /etc/kubernetes/cni/multus.yaml + +- name: Multus | Apply Multus configuration to the cluster + command: | + kubectl apply -f /etc/kubernetes/cni/multus.yaml diff --git a/builtin/core/roles/cni/multus/templates/multus.yaml b/builtin/core/roles/cni/multus/templates/multus.yaml new file mode 100644 index 000000000..d657c9a05 --- /dev/null +++ b/builtin/core/roles/cni/multus/templates/multus.yaml @@ -0,0 +1,206 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: network-attachment-definitions.k8s.cni.cncf.io +spec: + group: k8s.cni.cncf.io + scope: Namespaced + names: + plural: network-attachment-definitions + singular: network-attachment-definition + kind: NetworkAttachmentDefinition + shortNames: + - net-attach-def + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing + Working Group to express the intent for attaching pods to one or more logical or physical + networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec' + type: object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this represen + tation of an object. Servers should convert recognized schemas to the + latest internal value, and may reject unrecognized values. More info: + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment' + type: object + properties: + config: + description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration' + type: string +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: multus +rules: + - apiGroups: ["k8s.cni.cncf.io"] + resources: + - '*' + verbs: + - '*' + - apiGroups: + - "" + resources: + - pods + - pods/status + verbs: + - get + - update + - apiGroups: + - "" + - events.k8s.io + resources: + - events + verbs: + - create + - patch + - update +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: multus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: multus +subjects: + - kind: ServiceAccount + name: multus + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: multus + namespace: kube-system +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: multus-cni-config + namespace: kube-system + labels: + tier: node + app: multus +data: + # NOTE: If you'd prefer to manually apply a configuration file, you may create one here. + # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod + # change the "args" line below from + # - "--multus-conf-file=auto" + # to: + # "--multus-conf-file=/tmp/multus-conf/70-multus.conf" + # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the + # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet. + cni-conf.json: | + { + "name": "multus-cni-network", + "type": "multus", + "capabilities": { + "portMappings": true + }, + "delegates": [ + { + "cniVersion": "0.3.1", + "name": "default-cni-network", + "plugins": [ + { + "type": "flannel", + "name": "flannel.1", + "delegate": { + "isDefaultGateway": true, + "hairpinMode": true + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + } + ] + } + ], + "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig" + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-multus-ds + namespace: kube-system + labels: + tier: node + app: multus + name: multus +spec: + selector: + matchLabels: + name: multus + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + tier: node + app: multus + name: multus + spec: + hostNetwork: true + tolerations: + - operator: Exists + effect: NoSchedule + serviceAccountName: multus + containers: + - name: kube-multus + image: {{ .cni.multus.image.registry }}/{{ .cni.multus.image.repository }}:{{ .cni.multus.image.tag }} + command: ["/entrypoint.sh"] + args: + - "--multus-conf-file=auto" + - "--cni-version=0.3.1" + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: true + volumeMounts: + - name: cni + mountPath: /host/etc/cni/net.d + - name: cnibin + mountPath: /host/opt/cni/bin + - name: multus-cfg + mountPath: /tmp/multus-conf + terminationGracePeriodSeconds: 10 + volumes: + - name: cni + hostPath: + path: /etc/cni/net.d + - name: cnibin + hostPath: + path: /opt/cni/bin + - name: multus-cfg + configMap: + name: multus-cni-config + items: + - key: cni-conf.json + path: 70-multus.conf \ No newline at end of file diff --git a/builtin/core/roles/cri/containerd/defaults/main.yaml b/builtin/core/roles/cri/containerd/defaults/main.yaml new file mode 100644 index 000000000..ba5501cef --- /dev/null +++ b/builtin/core/roles/cri/containerd/defaults/main.yaml @@ -0,0 +1,3 @@ +cri: + containerd: + data_root: /var/lib/containerd \ No newline at end of file diff --git a/builtin/core/roles/cri/containerd/files/containerd.service b/builtin/core/roles/cri/containerd/files/containerd.service new file mode 100644 index 000000000..5f67110ab --- /dev/null +++ b/builtin/core/roles/cri/containerd/files/containerd.service @@ -0,0 +1,26 @@ +[Unit] +Description=containerd container runtime +Documentation=https://containerd.io +After=network.target local-fs.target + +[Service] +ExecStartPre=-/sbin/modprobe overlay +ExecStart=/usr/local/bin/containerd + +Type=notify +Delegate=yes +KillMode=process +Restart=always +RestartSec=5 +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNPROC=infinity +LimitCORE=infinity +LimitNOFILE=1048576 +# Comment TasksMax if your systemd version does not supports it. +# Only systemd 226 and above support this version. +TasksMax=infinity +OOMScoreAdjust=-999 + +[Install] +WantedBy=multi-user.target diff --git a/builtin/core/roles/cri/containerd/tasks/main.yaml b/builtin/core/roles/cri/containerd/tasks/main.yaml new file mode 100644 index 000000000..cfb9c04bc --- /dev/null +++ b/builtin/core/roles/cri/containerd/tasks/main.yaml @@ -0,0 +1,70 @@ +--- +- name: Containerd | Verify if runc is installed on the system + ignore_errors: true + command: runc --version + register: runc_install_version + +- name: Containerd | Ensure the runc binary is present on the remote node + when: or (.runc_install_version.error | empty | not) (.runc_install_version.stdout | contains (printf "runc version %s\n" (.cri.runc_version | default "" | trimPrefix "v" )) | not) + copy: + src: >- + {{ .binary_dir }}/runc/{{ .cri.runc_version }}/{{ .binary_type }}/runc.{{ .binary_type }} + dest: /usr/local/bin/runc + mode: 0755 + +- name: Containerd | Check if containerd is installed on the system + ignore_errors: true + command: containerd --version + register: containerd_install_version + +- name: Containerd | Install and configure containerd if not present or version mismatch + when: or (.containerd_install_version.error | empty | not) (.containerd_install_version.stdout | contains (printf " %s " .cri.containerd_version) | not) + block: + - name: Containerd | Copy containerd binary archive to the remote node + copy: + src: >- + {{ .binary_dir }}/containerd/{{ .cri.containerd_version }}/{{ .binary_type }}/containerd-{{ .cri.containerd_version | default "" | trimPrefix "v" }}-linux-{{ .binary_type }}.tar.gz + dest: >- + {{ .tmp_dir }}/containerd-{{ .cri.containerd_version | default "" | trimPrefix "v" }}-linux-{{ .binary_type }}.tar.gz + - name: Containerd | Extract containerd binaries to /usr/local/bin + command: | + tar -xvf {{ .tmp_dir }}/containerd-{{ .cri.containerd_version | default "" | trimPrefix "v" }}-linux-{{ .binary_type }}.tar.gz --strip-components=1 -C /usr/local/bin/ + - name: Containerd | Generate the containerd configuration file + template: + src: config.toml + dest: /etc/containerd/config.toml + - name: Containerd | Deploy the containerd systemd service file + copy: + src: containerd.service + dest: /etc/systemd/system/containerd.service + - name: Containerd | Start and enable the containerd service + command: | + systemctl daemon-reload && systemctl start containerd.service && systemctl enable containerd.service + +- name: Containerd | Synchronize image registry TLS certificates to the remote node + block: + - name: Containerd | Copy image registry CA certificate to the remote node + when: .image_registry.auth.ca_file | empty | not + copy: + src: >- + {{ .image_registry.auth.ca_file }} + dest: >- + /etc/containerd/certs.d/{{ .image_registry.auth.registry }}/ca.crt + - name: Containerd | Copy image registry server certificate to the remote node + when: + - .image_registry.auth.cert_file | empty | not + - .image_registry.auth.cert_file | fileExist + copy: + src: >- + {{ .image_registry.auth.cert_file }} + dest: >- + /etc/containerd/certs.d/{{ .image_registry.auth.registry }}/server.crt + - name: Containerd | Copy image registry server key to the remote node + when: + - .image_registry.auth.key_file | empty | not + - .image_registry.auth.key_file | fileExist + copy: + src: >- + {{ .image_registry.auth.key_file }} + dest: >- + /etc/containerd/certs.d/{{ .image_registry.auth.registry }}/server.key diff --git a/builtin/core/roles/cri/containerd/templates/config.toml b/builtin/core/roles/cri/containerd/templates/config.toml new file mode 100644 index 000000000..3d3afe982 --- /dev/null +++ b/builtin/core/roles/cri/containerd/templates/config.toml @@ -0,0 +1,93 @@ +version = 2 + +root = "{{ .cri.containerd.data_root | default "/var/lib/containerd" }}" +state = "/run/containerd" + +[grpc] + address = "/run/containerd/containerd.sock" + uid = 0 + gid = 0 + max_recv_message_size = 16777216 + max_send_message_size = 16777216 + +[ttrpc] + address = "" + uid = 0 + gid = 0 + +[debug] + address = "" + uid = 0 + gid = 0 + level = "" + +[metrics] + address = "" + grpc_histogram = false + +[cgroup] + path = "" + +[timeouts] + "io.containerd.timeout.shim.cleanup" = "5s" + "io.containerd.timeout.shim.load" = "5s" + "io.containerd.timeout.shim.shutdown" = "3s" + "io.containerd.timeout.task.state" = "2s" + +[plugins] + [plugins."io.containerd.grpc.v1.cri"] + sandbox_image = "{{ .cri.sandbox_image.registry }}/{{ .cri.sandbox_image.repository }}:{{ .cri.sandbox_image.tag }}" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = {{ if .cri.cgroup_driver | eq "systemd" }}true{{ else }}false{{ end }} + [plugins."io.containerd.grpc.v1.cri".cni] + bin_dir = "/opt/cni/bin" + conf_dir = "/etc/cni/net.d" + max_conf_num = 1 + conf_template = "" + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] +{{- if .cri.registry.mirrors | empty | not }} + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = {{ .cri.registry.mirrors | toJson }} +{{- end }} +{{- range .cri.registry.insecure_registries }} + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ . }}"] + endpoint = ["http://{{ . }}"] +{{- end }} +{{- if or (.cri.registry.auths | empty | not) (.groups.image_registry | default list | empty | not) }} + [plugins."io.containerd.grpc.v1.cri".registry.configs] +{{- end }} + [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .image_registry.auth.registry }}".auth] + username = "{{ .image_registry.auth.username }}" + password = "{{ .image_registry.auth.password }}" + [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .image_registry.auth.registry }}".tls] +{{- if .image_registry.auth.ca_file | empty | not }} + ca_file = "/etc/containerd/certs.d/{{ .image_registry.auth.registry }}/ca.crt" +{{- end }} +{{- if .image_registry.auth.cert_file | empty | not }} + cert_file = "/etc/containerd/certs.d/{{ .image_registry.auth.registry }}/server.crt" +{{- end }} +{{- if .image_registry.auth.key_file | empty | not }} + key_file = "/etc/containerd/certs.d/{{ .image_registry.auth.registry }}/server.key" +{{- end }} + insecure_skip_verify = {{ .image_registry.auth.insecure | default true }} +{{- if .cri.registry.auths | empty | not }} + {{- range .cri.registry.auths }} + [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .repo }}".auth] + username = "{{ .username }}" + password = "{{ .password }}" + [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .repo }}".tls] + {{- if .ca_file }} + ca_file = {{ .ca_file }} + {{- end }} + {{- if .crt_file }} + cert_file = {{ .crt_file }} + {{- end }} + {{- if .key_file }} + key_file = {{ .key_file }} + {{- end }} + insecure_skip_verify = {{ .insecure | default true }} + {{- end }} +{{- end }} diff --git a/builtin/core/roles/cri/crictl/tasks/main.yaml b/builtin/core/roles/cri/crictl/tasks/main.yaml new file mode 100644 index 000000000..bbcafc35e --- /dev/null +++ b/builtin/core/roles/cri/crictl/tasks/main.yaml @@ -0,0 +1,22 @@ +--- +- name: Crictl | Verify if crictl is installed on the system + ignore_errors: true + command: crictl --version + register: crictl_install_version + +- name: Crictl | Install and configure crictl if not present or version mismatch + when: or (.crictl_install_version.error | empty | not) (.crictl_install_version.stdout | ne (printf "crictl version %s" .cri.crictl_version)) + block: + - name: Crictl | Copy crictl binary archive to the remote node + copy: + src: >- + {{ .binary_dir }}/crictl/{{ .cri.crictl_version }}/{{ .binary_type }}/crictl-{{ .cri.crictl_version }}-linux-{{ .binary_type }}.tar.gz + dest: >- + {{ .tmp_dir }}/crictl-{{ .cri.crictl_version }}-linux-{{ .binary_type }}.tar.gz + - name: Crictl | Extract crictl binary to /usr/local/bin + command: | + tar -xvf {{ .tmp_dir }}/crictl-{{ .cri.crictl_version }}-linux-{{ .binary_type }}.tar.gz -C /usr/local/bin/ + - name: Crictl | Generate crictl configuration file + template: + src: crictl.yaml + dest: /etc/crictl.yaml diff --git a/builtin/core/roles/cri/crictl/templates/crictl.yaml b/builtin/core/roles/cri/crictl/templates/crictl.yaml new file mode 100644 index 000000000..9a8544e69 --- /dev/null +++ b/builtin/core/roles/cri/crictl/templates/crictl.yaml @@ -0,0 +1,5 @@ +runtime-endpoint: {{ .cri.cri_socket }} +image-endpoint: {{ .cri.cri_socket }} +timeout: 5 +debug: false +pull-image-on-create: false diff --git a/builtin/core/roles/cri/docker/defaults/main.yaml b/builtin/core/roles/cri/docker/defaults/main.yaml new file mode 100644 index 000000000..888b6f795 --- /dev/null +++ b/builtin/core/roles/cri/docker/defaults/main.yaml @@ -0,0 +1,3 @@ +cri: + docker: + data_root: /var/lib/docker \ No newline at end of file diff --git a/builtin/core/roles/cri/docker/files/containerd.service b/builtin/core/roles/cri/docker/files/containerd.service new file mode 100644 index 000000000..5f67110ab --- /dev/null +++ b/builtin/core/roles/cri/docker/files/containerd.service @@ -0,0 +1,26 @@ +[Unit] +Description=containerd container runtime +Documentation=https://containerd.io +After=network.target local-fs.target + +[Service] +ExecStartPre=-/sbin/modprobe overlay +ExecStart=/usr/local/bin/containerd + +Type=notify +Delegate=yes +KillMode=process +Restart=always +RestartSec=5 +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNPROC=infinity +LimitCORE=infinity +LimitNOFILE=1048576 +# Comment TasksMax if your systemd version does not supports it. +# Only systemd 226 and above support this version. +TasksMax=infinity +OOMScoreAdjust=-999 + +[Install] +WantedBy=multi-user.target diff --git a/builtin/core/roles/cri/docker/files/docker.service b/builtin/core/roles/cri/docker/files/docker.service new file mode 100644 index 000000000..929b8a436 --- /dev/null +++ b/builtin/core/roles/cri/docker/files/docker.service @@ -0,0 +1,47 @@ +[Unit] +Description=Docker Application Container Engine +Documentation=https://docs.docker.com +# After=network-online.target firewalld.service containerd.service +# Wants=network-online.target +# Requires=docker.socket containerd.service + +[Service] +Type=notify +# the default is not to use systemd for cgroups because the delegate issues still +# exists and systemd currently does not support the cgroup feature set required +# for containers run by docker +ExecStart=/usr/local/bin/dockerd --containerd=/run/containerd/containerd.sock +ExecReload=/bin/kill -s HUP $MAINPID +TimeoutSec=0 +RestartSec=2 +Restart=always + +# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229. +# Both the old, and new location are accepted by systemd 229 and up, so using the old location +# to make them work for either version of systemd. +StartLimitBurst=3 + +# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230. +# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make +# this option work for either version of systemd. +StartLimitInterval=60s + +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity + +# Comment TasksMax if your systemd version does not support it. +# Only systemd 226 and above support this option. +TasksMax=infinity + +# set delegate yes so that systemd does not reset the cgroups of docker containers +Delegate=yes + +# kill only the docker process, not all processes in the cgroup +KillMode=process +OOMScoreAdjust=-500 + +[Install] +WantedBy=multi-user.target diff --git a/builtin/core/roles/cri/docker/tasks/cridockerd.yaml b/builtin/core/roles/cri/docker/tasks/cridockerd.yaml new file mode 100644 index 000000000..cfbe4f821 --- /dev/null +++ b/builtin/core/roles/cri/docker/tasks/cridockerd.yaml @@ -0,0 +1,25 @@ +--- +- name: Cridockerd | Check if cri-dockerd is installed on the system + ignore_errors: true + command: cri-dockerd --version + register: cridockerd_install_version + +- name: Cridockerd | Install and configure cri-dockerd if not present or version mismatch + when: or (.cridockerd_install_version.error | empty | not) (.cridockerd_install_version.stdout | hasPrefix (printf "cri-dockerd %s " .cri.cridockerd_version) | not) + block: + - name: Cridockerd | Copy cri-dockerd binary archive to the remote node + copy: + src: >- + {{ .binary_dir }}/cri-dockerd/{{ .cri.cridockerd_version }}/{{ .binary_type }}/cri-dockerd-{{ .cri.cridockerd_version | default "" | trimPrefix "v" }}.{{ .binary_type }}.tgz + dest: >- + {{ .tmp_dir }}/cri-dockerd-{{ .cri.cridockerd_version | default "" | trimPrefix "v" }}.{{ .binary_type }}.tgz + - name: Cridockerd | Extract cri-dockerd binary to /usr/local/bin + command: | + tar -xvf {{ .tmp_dir }}/cri-dockerd-{{ .cri.cridockerd_version | default "" | trimPrefix "v" }}.{{ .binary_type }}.tgz --strip-components=1 -C /usr/local/bin/ + - name: Cridockerd | Generate cri-dockerd systemd service file + template: + src: cri-dockerd.service + dest: /etc/systemd/system/cri-dockerd.service + - name: Cridockerd | Start and enable the cri-dockerd service + command: | + systemctl daemon-reload && systemctl start cri-dockerd.service && systemctl enable cri-dockerd.service diff --git a/builtin/core/roles/cri/docker/tasks/main.yaml b/builtin/core/roles/cri/docker/tasks/main.yaml new file mode 100644 index 000000000..6de85e0cb --- /dev/null +++ b/builtin/core/roles/cri/docker/tasks/main.yaml @@ -0,0 +1,68 @@ +--- + +- name: Docker | Check if Docker is installed on the system + ignore_errors: true + command: docker --version + register: docker_install_version + +- name: Docker | Install and configure Docker if not present or version mismatch + when: or (.docker_install_version.error | empty | not) (.docker_install_version.stdout | hasPrefix (printf "Docker version %s," .cri.docker_version) | not) + block: + - name: Docker | Copy Docker binary archive to the remote node + copy: + src: >- + {{ .binary_dir }}/docker/{{ .cri.docker_version }}/{{ .binary_type }}/docker-{{ .cri.docker_version }}.tgz + dest: >- + {{ .tmp_dir }}/docker-{{ .cri.docker_version }}.tgz + - name: Docker | Extract Docker binaries to /usr/local/bin + command: | + tar -C /usr/local/bin/ --strip-components=1 -xvf {{ .tmp_dir }}/docker-{{ .cri.docker_version }}.tgz --wildcards 'docker/*' + - name: Docker | Generate Docker configuration file + template: + src: daemon.json + dest: /etc/docker/daemon.json + - name: Docker | Deploy the Docker systemd service file + copy: + src: docker.service + dest: /etc/systemd/system/docker.service + - name: Docker | Deploy the containerd systemd service file + copy: + src: containerd.service + dest: /etc/systemd/system/containerd.service + - name: Docker | Start and enable Docker and containerd services + command: | + systemctl daemon-reload && systemctl start containerd.service && systemctl enable containerd.service + systemctl daemon-reload && systemctl start docker.service && systemctl enable docker.service + +- name: Docker | Synchronize image registry TLS certificates to the remote node + block: + - name: Docker | Copy image registry CA certificate to the remote node + when: .image_registry.auth.ca_file | empty | not + copy: + src: >- + {{ .image_registry.auth.ca_file }} + dest: >- + /etc/docker/certs.d/{{ .image_registry.auth.registry }}/ca.crt + - name: Docker | Copy image registry server certificate to the remote node + when: + - .image_registry.auth.cert_file | empty | not + - .image_registry.auth.cert_file | fileExist + copy: + src: >- + {{ .image_registry.auth.cert_file }} + dest: >- + /etc/docker/certs.d/{{ .image_registry.auth.registry }}/client.cert + - name: Docker | Copy image registry server key to the remote node + when: + - .image_registry.auth.key_file | empty | not + - .image_registry.auth.key_file | fileExist + copy: + src: >- + {{ .image_registry.auth.key_file }} + dest: >- + /etc/docker/certs.d/{{ .image_registry.auth.registry }}/client.key + +# Docker | Install cri-dockerd if required for Kubernetes >= v1.24.0 +- include_tasks: cridockerd.yaml + when: + - .kubernetes.kube_version | semverCompare ">=v1.24.0" \ No newline at end of file diff --git a/builtin/core/roles/cri/docker/templates/cri-dockerd.service b/builtin/core/roles/cri/docker/templates/cri-dockerd.service new file mode 100644 index 000000000..244ef9a68 --- /dev/null +++ b/builtin/core/roles/cri/docker/templates/cri-dockerd.service @@ -0,0 +1,36 @@ +[Unit] +Description=CRI Interface for Docker Application Container Engine +Documentation=https://docs.mirantis.com + +[Service] +Type=notify +ExecStart=/usr/local/bin/cri-dockerd --pod-infra-container-image "{{ .cri.sandbox_image.registry }}/{{ .cri.sandbox_image.repository }}:{{ .cri.sandbox_image.tag }}" +ExecReload=/bin/kill -s HUP $MAINPID +TimeoutSec=0 +RestartSec=2 +Restart=always + +# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229. +# Both the old, and new location are accepted by systemd 229 and up, so using the old location +# to make them work for either version of systemd. +StartLimitBurst=3 + +# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230. +# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make +# this option work for either version of systemd. +StartLimitInterval=60s + +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity + +# Comment TasksMax if your systemd version does not support it. +# Only systemd 226 and above support this option. +TasksMax=infinity +Delegate=yes +KillMode=process + +[Install] +WantedBy=multi-user.target diff --git a/builtin/core/roles/cri/docker/templates/daemon.json b/builtin/core/roles/cri/docker/templates/daemon.json new file mode 100644 index 000000000..1e914e17c --- /dev/null +++ b/builtin/core/roles/cri/docker/templates/daemon.json @@ -0,0 +1,21 @@ +{ + "log-opts": { + "max-size": "5m", + "max-file":"3" + }, +{{- if .cri.docker.data_root }} + "data-root": "{{ .cri.docker.data_root }}", +{{- end }} +{{- if .cri.registry.mirrors }} + "registry-mirrors": {{ .cri.registry.mirrors | toJson }}, +{{- end }} +{{- $insecure_registries := .cri.registry.insecure_registries | default list -}} +{{- if .image_registry.auth.insecure -}} + {{- $insecure_registries = append $insecure_registries .image_registry.auth.registry -}} +{{- end -}} + "insecure-registries": {{ $insecure_registries | toJson }}, + {{- if .cri.docker.bridge_ip }} + "bip": "{{ .cri.docker.bridge_ip }}", +{{- end }} + "exec-opts": ["native.cgroupdriver={{ .cri.cgroup_driver }}"] +} diff --git a/builtin/core/roles/cri/meta/main.yaml b/builtin/core/roles/cri/meta/main.yaml new file mode 100644 index 000000000..34b408999 --- /dev/null +++ b/builtin/core/roles/cri/meta/main.yaml @@ -0,0 +1,8 @@ +dependencies: + - role: cri/crictl + + - role: cri/docker + when: .cri.container_manager | eq "docker" + + - role: cri/containerd + when: .cri.container_manager | eq "containerd" \ No newline at end of file diff --git a/builtin/core/roles/defaults/defaults/main/01-cluster_require.yaml b/builtin/core/roles/defaults/defaults/main/01-cluster_require.yaml new file mode 100644 index 000000000..6e27c0c2f --- /dev/null +++ b/builtin/core/roles/defaults/defaults/main/01-cluster_require.yaml @@ -0,0 +1,36 @@ +# Cluster parameter boundaries +cluster_require: + # Maximum etcd WAL fsync duration for 99th percentile (in nanoseconds) + etcd_disk_wal_fysnc_duration_seconds: 10000000 + # Allow installation on unsupported Linux distributions + allow_unsupported_distribution_setup: false + # Supported operating system distributions + supported_os_distributions: + - ubuntu + - '"ubuntu"' + - centos + - '"centos"' + # Required network plugins + require_network_plugin: ['calico', 'flannel', 'cilium', 'hybridnet', 'kube-ovn'] + # Minimum supported Kubernetes version + kube_version_min_required: v1.23.0 + # Minimum memory (in MB) required for each control plane node + # Must be greater than or equal to minimal_master_memory_mb + minimal_master_memory_mb: 10 + # Minimum memory (in MB) required for each worker node + # Must be greater than or equal to minimal_node_memory_mb + minimal_node_memory_mb: 10 + # Supported etcd deployment types + require_etcd_deployment_type: ['internal', 'external'] + # Supported container runtimes + require_container_manager: ['docker', 'containerd'] + # Minimum required version of containerd + containerd_min_version_required: v1.6.0 + # Supported CPU architectures + supported_architectures: + - amd64 + - x86_64 + - arm64 + - aarch64 + # Minimum required Linux kernel version + min_kernel_version: 4.9.17 \ No newline at end of file diff --git a/builtin/core/roles/defaults/defaults/main/01-main.yaml b/builtin/core/roles/defaults/defaults/main/01-main.yaml new file mode 100644 index 000000000..d921df139 --- /dev/null +++ b/builtin/core/roles/defaults/defaults/main/01-main.yaml @@ -0,0 +1,39 @@ +work_dir: /root/kubekey +binary_dir: >- + {{ .work_dir }}/kubekey +scripts_dir: >- + {{ .binary_dir }}/scripts +tmp_dir: /tmp/kubekey + +# Mapping of common machine architecture names to their standard forms +transform_architectures: + amd64: + - amd64 + - x86_64 + arm64: + - arm64 + - aarch64 + + +# Enable enhanced security features for stricter cluster security requirements. +security_enhancement: false + +# Enable Kubernetes audit logging. +# Audit logs record and track critical operations within the cluster, helping administrators monitor security events, troubleshoot issues, and meet compliance requirements (e.g., SOC2, ISO 27001). +audit: false + +# When removing a node, also uninstall the node's container runtime (CRI), such as Docker or containerd. +deleteCRI: true + +# When removing a node, also uninstall etcd from the node. +deleteETCD: true + +# When removing a node, restore the node's DNS configuration. +deleteDNS: true + +# When removing a node, also uninstall any private image registry (such as Harbor or registry) installed on the node. +# This is typically used in conjunction with nodes defined in inventory.groups.image_registry. +deleteImageRegistry: false + +# image_manifests: List of container images to be synchronized to the private registry +image_manifests: [] \ No newline at end of file diff --git a/builtin/core/roles/defaults/defaults/main/02-certs.yaml b/builtin/core/roles/defaults/defaults/main/02-certs.yaml new file mode 100644 index 000000000..a7fca7f48 --- /dev/null +++ b/builtin/core/roles/defaults/defaults/main/02-certs.yaml @@ -0,0 +1,49 @@ +# Certificate generation configuration +# The following certificates will be generated: +# - etcd certificates +# - Kubernetes cluster certificates (replacing the CA certificate generated by kubeadm, which is limited to a 10-year validity) +# - Image registry certificates (for Harbor and similar registries) + +# Certificate chain structure: +# CA (self-signed or provided) +# |- etcd.cert +# |- etcd.key +# | +# |- image_registry.cert +# |- image_registry.key +# | +# |- kubernetes.cert +# |- kubernetes.key +# | |- kubeadm uses this to generate server certificates (kube-apiserver certificate) +# |- front-proxy.cert +# |- front-proxy.key +# | +# |- image-registry.cert +# |- image-registry.key + +certs: + # CA certificate settings + ca: + # CA certificate expiration time + date: 87600h + # Certificate generation policy: + # IfNotPresent: Validate the certificate if it exists; generate a self-signed certificate only if it does not exist + gen_cert_policy: IfNotPresent + kubernetes_ca: + date: 87600h + # How to generate the certificate file. Supported values: IfNotPresent, Always + gen_cert_policy: IfNotPresent + front_proxy_ca: + date: 87600h + # How to generate the certificate file. Supported values: IfNotPresent, Always + gen_cert_policy: IfNotPresent + # etcd certificate + etcd: + date: 87600h + # How to generate the certificate file. Supported values: IfNotPresent, Always + gen_cert_policy: IfNotPresent + # image_registry certificate + image_registry: + date: 87600h + # How to generate the certificate file. Supported values: IfNotPresent, Always + gen_cert_policy: IfNotPresent \ No newline at end of file diff --git a/builtin/core/roles/defaults/defaults/main/02-image_registry.yaml b/builtin/core/roles/defaults/defaults/main/02-image_registry.yaml new file mode 100644 index 000000000..636b83811 --- /dev/null +++ b/builtin/core/roles/defaults/defaults/main/02-image_registry.yaml @@ -0,0 +1,79 @@ +# In an online environment (when image_registry.auth.registry is empty), images are pulled directly from their original registries to the cluster. +# In an offline environment (when image_registry.auth.registry is set), images are first pulled from the source registry, cached locally, pushed to a private registry (such as Harbor), and then used by the cluster. + +image_registry: + # Specify which image registry to install. Supported values: harbor, docker-registry + # If left empty, no image registry will be installed (assumes an existing registry is already available). + type: "" + ha_vip: "" + # Directory where images to be pushed to the registry are stored. + # Path for storing offline images + images_dir: >- + {{ .tmp_dir }}/images/ + # Image registry authentication settings + auth: + registry: >- + {{- if .image_registry.type | empty | not }} + {{- if .image_registry.ha_vip | empty | not -}} + {{ .image_registry.ha_vip }} + {{- else if .groups.image_registry | default list | empty | not -}} + {{- $internalIPv4 := index .hostvars (.groups.image_registry | default list | first) "internal_ipv4" | default "" -}} + {{- $internalIPv6 := index .hostvars (.groups.image_registry | default list | first) "internal_ipv6" | default "" -}} + {{- if $internalIPv4 | empty | not -}} + {{ $internalIPv4 }} + {{- else if $internalIPv6 | empty | not -}} + {{ $internalIPv6 }} + {{- end -}} + {{- end -}} + {{- end -}} + username: admin + password: Harbor12345 + insecure: >- + {{- if .image_registry.type | empty -}} + true + {{- end -}} + ca_file: >- + {{- if .image_registry.type | empty | not -}} + {{ .binary_dir }}/pki/root.crt + {{- end -}} + cert_file: >- + {{- if .image_registry.type | empty | not -}} + {{ .binary_dir }}/pki/image_registry.crt + {{- end -}} + key_file: >- + {{- if .image_registry.type | empty | not -}} + {{ .binary_dir }}/pki/image_registry.key + {{- end -}} + # Registry endpoint for images from docker.io + dockerio_registry: >- + {{- if .image_registry.auth.registry | empty | not -}} + {{ .image_registry.auth.registry }} + {{- else -}} + docker.io + {{- end -}} + + # Registry endpoint for images from quay.io + quayio_registry: >- + {{- if .image_registry.auth.registry | empty | not -}} + {{ .image_registry.auth.registry }} + {{- else -}} + quay.io + {{- end -}} + + # Registry endpoint for images from ghcr.io + ghcrio_registry: >- + {{- if .image_registry.auth.registry | empty | not -}} + {{ .image_registry.auth.registry }} + {{- else -}} + ghcr.io + {{- end -}} + + # ========== image registry ========== + # keepalived image tag. Used for load balancing when there are multiple image registry nodes. + keepalived_version: 2.0.20 + # ========== image registry: harbor ========== + # harbor image tag + harbor_version: v2.10.1 + # ========== image registry: docker-registry ========== + # docker-registry image tag + docker_registry_version: 2.8.3 diff --git a/builtin/core/roles/defaults/defaults/main/02-native.yaml b/builtin/core/roles/defaults/defaults/main/02-native.yaml new file mode 100644 index 000000000..2d80e0d2e --- /dev/null +++ b/builtin/core/roles/defaults/defaults/main/02-native.yaml @@ -0,0 +1,25 @@ +# Essential operating system configuration settings +native: + ntp: + # List of NTP servers used for system time synchronization + servers: + - "cn.pool.ntp.org" + # Toggle to enable or disable the NTP service + enabled: true + # System timezone configuration + timezone: Asia/Shanghai + + # NFS service configuration for nodes assigned the 'nfs' role in the inventory + nfs: + # Directories to be shared via NFS + share_dir: + - /share/ + # Whether to set the node's hostname to the value defined in inventory.hosts. + set_hostname: true + # List of DNS configuration files to update on each node. + # This ensures that, during cluster installation, critical hostnames can be resolved locally even if no DNS service is available. + # For example: + # [control_plane_endpoint of master node] -> master node IP + # [hostname of the node being installed] -> corresponding node IP + localDNS: + - /etc/hosts \ No newline at end of file diff --git a/builtin/core/roles/defaults/defaults/main/03-kubernetes.yaml b/builtin/core/roles/defaults/defaults/main/03-kubernetes.yaml new file mode 100644 index 000000000..e23f8797d --- /dev/null +++ b/builtin/core/roles/defaults/defaults/main/03-kubernetes.yaml @@ -0,0 +1,116 @@ +kubernetes: + # Name of the cluster to be installed + cluster_name: kubekey + + # Kubernetes version to deploy + kube_version: v1.33.1 + # helm binary + helm_version: v3.18.5 + + # Image repository for built-in Kubernetes images + image_repository: >- + {{ .image_registry.dockerio_registry }}/kubesphere + + # Kubernetes network configuration + # kube-apiserver pod parameters + apiserver: + port: 6443 + certSANs: [] + extra_args: + # Example: feature-gates: ExpandCSIVolumes=true,CSIStorageCapacity=true,RotateKubeletServerCertificate=true + + # kube-controller-manager pod parameters + controller_manager: + extra_args: + cluster-signing-duration: 87600h + # Example: feature-gates: ExpandCSIVolumes=true,CSIStorageCapacity=true,RotateKubeletServerCertificate=true + + # kube-scheduler pod parameters + scheduler: + extra_args: + # Example: feature-gates: ExpandCSIVolumes=true,CSIStorageCapacity=true,RotateKubeletServerCertificate=true + + # kube-proxy pod parameters + kube_proxy: + enabled: true + # Supported proxy modes: ipvs, iptables + mode: "ipvs" + config: + iptables: + masqueradeAll: false + masqueradeBit: 14 + minSyncPeriod: 0s + syncPeriod: 30s + + # kubelet service parameters + kubelet: + max_pod: 110 + pod_pids_limit: 10000 +# feature_gates: + container_log_max_size: 5Mi + container_log_max_files: 3 +# extra_args: + + # Specify a stable IP address or DNS name for the control plane endpoint. + # For high availability, it is recommended to set control_plane_endpoint to a DNS name. + # Configuration guidance: + # 1. If a DNS name is available: + # - Set control_plane_endpoint to that DNS name and ensure it resolves to all control plane node IPs. + # 2. If no DNS name is available: + # - You can set a DNS name now and add the resolution later. + # - Add the resolution to each node's local DNS file, for example: + # {{ vip }} {{ control_plane_endpoint }} + # - If you have a VIP (Virtual IP): + # Deploy kube-vip on control plane nodes to map the VIP to the actual node IPs. + # - If you do not have a VIP: + # Deploy HAProxy on worker nodes, use a fixed IP (such as 127.0.0.2) as the VIP, and forward to all control plane node IPs. + # + # For non-HA scenarios (manual configuration only, not automatically installed): + # You can set the VIP to the IP of a single control plane node. + control_plane_endpoint: + host: lb.kubesphere.local + port: "{{ .kubernetes.apiserver.port }}" + # Supported types: local, kube_vip, haproxy + # When type is local, configure as follows: + # - On control-plane nodes: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }} + # - On worker nodes: {{ .init_kubernetes_node }} {{ .kubernetes.control_plane_endpoint.host }} + type: local + kube_vip: + # The IP address of the node's network interface (e.g., "eth0"). + address: "" + # Supported modes: ARP, BGP + mode: ARP + image: + registry: >- + {{ .dockerio_registry }} + repository: plndr/kube-vip + tag: v0.7.2 + haproxy: + # The IP address on the node's "lo" (loopback) interface. + address: 127.0.0.1 + health_port: 8081 + image: + registry: >- + {{ .dockerio_registry }} + repository: library/haproxy + tag: 2.9.6-alpine + + # Whether to automatically renew Kubernetes certificates + certs: + # There are three ways to provide the Kubernetes CA (Certificate Authority) files: + # 1. kubeadm: Leave ca_cert and ca_key empty, and kubeadm will generate them automatically. These certificates are valid for 10 years and will not change. + # 2. kubekey: Set ca_cert to {{ .binary_dir }}/pki/ca.cert and ca_key to {{ .binary_dir }}/pki/ca.key. + # These certificates are generated by kubekey, valid for 10 years, and can be updated via `cert.ca_date`. + # 3. Custom: Manually specify the absolute paths for ca_cert and ca_key to use your own CA files. + # + # To use custom CA files, fill in the absolute paths below. + # If left empty, the default behavior (kubeadm or kubekey) will be used. + ca_cert: "" + ca_key: "" + # The following fields are for the Kubernetes front-proxy CA certificate and key. + # To use custom front-proxy CA files, fill in the absolute paths below. + # If left empty, the default behavior will be used. + front_proxy_cert: "" + front_proxy_key: "" + # Automatically renew service certificates (Note: CA certificates cannot be renewed automatically) + renew: false diff --git a/builtin/core/roles/defaults/defaults/main/04-cni.yaml b/builtin/core/roles/defaults/defaults/main/04-cni.yaml new file mode 100644 index 000000000..6e963a56a --- /dev/null +++ b/builtin/core/roles/defaults/defaults/main/04-cni.yaml @@ -0,0 +1,41 @@ +cni: + # CNI plugin to use (equivalent to kubernetes.kube_network_plugin) + # Specify the network plugin to install for the cluster. Supported: calico, cilium, flannel, hybridnet, kubeovn, other + # kube_network_plugin: calico + type: calico + # Maximum number of pods supported per node + max_pods: 110 + # The complete Pod IP pool for the cluster. Supports IPv4, IPv6, and dual-stack. + pod_cidr: 10.233.64.0/18 + # IPv4 subnet mask length for pod allocation per node. Determines the size of each node's pod IP pool. + ipv4_mask_size: 24 + # IPv6 subnet mask length for pod allocation per node. + ipv6_mask_size: 64 + # The complete Service IP pool for the cluster. Supports IPv4, IPv6, and dual-stack. + service_cidr: 10.233.0.0/18 + + # Network enhancement plugin for multiple pod network interfaces (Multus) + multus: + # Enable or disable the network enhancement plugin + enabled: false + image: + registry: >- + {{ .image_registry.ghcrio_registry }} + repository: k8snetworkplumbingwg/multus-cni + tag: v4.3.0 + + # ========== cni ========== + # cni_plugins binary + # cni_plugins_version: v1.2.0 + # ========== cni: calico ========== + # calicoctl binary + calico_version: v3.28.2 + # ========== cni: cilium ========== + # cilium helm + cilium_version: 1.15.4 + # ========== cni: kubeovn ========== + # kubeovn helm + kubeovn_version: 1.13.0 + # ========== cni: hybridnet ========== + # hybridnet helm + hybridnet_version: 0.6.8 \ No newline at end of file diff --git a/builtin/core/roles/defaults/defaults/main/04-cri.yaml b/builtin/core/roles/defaults/defaults/main/04-cri.yaml new file mode 100644 index 000000000..7a96c18dd --- /dev/null +++ b/builtin/core/roles/defaults/defaults/main/04-cri.yaml @@ -0,0 +1,40 @@ +cri: + # Container runtime to use. Supported: containerd, docker + container_manager: containerd + # Cgroup driver for the container runtime. Supported: systemd, cgroupfs + cgroup_driver: systemd + # Pause/sandbox image configuration + sandbox_image: + registry: >- + {{ .image_registry.dockerio_registry }} + repository: kubesphere/pause + tag: "3.9" + # CRI socket endpoint for the selected container runtime + cri_socket: >- + {{- if .cri.container_manager | eq "containerd" -}} + unix:///var/run/containerd/containerd.sock + {{- else if and (.cri.container_manager | eq "docker") (.kubernetes.kube_version | semverCompare ">=v1.24.0") -}} + unix:///var/run/cri-dockerd.sock + {{- end -}} + + # Registry configuration for CRI, including mirrors, insecure registries, and authentication + registry: + mirrors: ["https://registry-1.docker.io"] + insecure_registries: [] + auths: [] + + # ========== cri ========== + # crictl binary + crictl_version: v1.33.0 + # ========== cri: docker ========== + # docker binary + docker_version: 24.0.7 + # docker-compose binary + dockercompose_version: v2.20.3 + # cridockerd. Required when kube_version is greater than 1.24 + cridockerd_version: v0.3.1 + # ========== cri: containerd ========== + # containerd binary + containerd_version: v1.7.6 + # runc binary + runc_version: v1.1.7 \ No newline at end of file diff --git a/builtin/core/roles/defaults/defaults/main/04-etcd.yaml b/builtin/core/roles/defaults/defaults/main/04-etcd.yaml new file mode 100644 index 000000000..b7a193563 --- /dev/null +++ b/builtin/core/roles/defaults/defaults/main/04-etcd.yaml @@ -0,0 +1,44 @@ +# etcd service configuration +etcd: + # etcd supports two deployment types: + # - external: Use an external etcd cluster. + # - internal: Deploy etcd as static Pods within the cluster. + deployment_type: external + etcd_version: v3.5.11 + image: + registry: >- + {{ .image_registry.dockerio_registry }} + repository: kubesphere/etcd + tag: "{{ .etcd.etcd_version }}" + # endpoints: ["https://127.1.1.1:2379"] + # Environment variables for etcd service + env: + election_timeout: 5000 + heartbeat_interval: 250 + compaction_retention: 8 + snapshot_count: 10000 + data_dir: /var/lib/etcd + token: k8s_etcd + # metrics: basic + # quota_backend_bytes: 100 + # max_request_bytes: 100 + # max_snapshots: 100 + # max_wals: 5 + # log_level: info + # unsupported_arch: arm64 + # etcd backup configuration + backup: + backup_dir: /var/lib/etcd-backup + keep_backup_number: 5 + etcd_backup_script: "backup.sh" + on_calendar: "*-*-* *:00/30:00" + # Enable etcd performance tuning (set to true to enable) + performance: false + # Enable etcd traffic prioritization (set to true to enable) + traffic_priority: false + ca_file: >- + {{ .binary_dir }}/pki/root.crt + cert_file: >- + {{ .binary_dir }}/pki/etcd.crt + key_file: >- + {{ .binary_dir }}/pki/etcd.key \ No newline at end of file diff --git a/builtin/core/roles/defaults/defaults/main/05-dns.yaml b/builtin/core/roles/defaults/defaults/main/05-dns.yaml new file mode 100644 index 000000000..684082c74 --- /dev/null +++ b/builtin/core/roles/defaults/defaults/main/05-dns.yaml @@ -0,0 +1,78 @@ +dns: + # ====== In-Cluster DNS Service Configuration ====== + # The DNS domain suffix used for all services and pods within the cluster. + dns_domain: cluster.local + # CoreDNS image settings + dns_image: + registry: >- + {{ .image_registry.dockerio_registry }} + repository: >- + kubesphere + tag: v1.11.1 + # NodeLocalDNS image settings + dns_cache_image: + registry: >- + {{ .image_registry.dockerio_registry }} + repository: kubesphere/k8s-dns-node-cache + tag: 1.24.0 + # The IP address assigned to the cluster DNS service + dns_service_ip: >- + {{ index (.cni.service_cidr | ipInCIDR) 2 }} + # The IP address NodeLocalDNS will bind to on each node + dns_cache_ip: 169.254.25.10 + + # CoreDNS pod configuration + coredns: + dns_etc_hosts: [] + # DNS zone matching configuration + zone_configs: + # Each entry defines which DNS zones to match. The default port is 53. + # ".": matches all DNS zones. + # "example.com": matches *.example.com using DNS server on port 53. + # "example.com:54": matches *.example.com using DNS server on port 54. + - zones: [".:53"] + additional_configs: + - errors + - ready + - prometheus :9153 + - loop + - reload + - loadbalance + cache: 30 + kubernetes: + zones: + - "{{ .dns.dns_domain }}" + # You can configure internal DNS message rewriting here if needed. +# rewrite: +# - rule: continue +# field: name +# type: exact +# value: "example.com example2.com" +# options: "" + forward: + # DNS query forwarding rules. + - from: "." + # Destination endpoints for forwarding. The 'to' syntax allows protocol specification. + to: ["/etc/resolv.conf"] + # Domains to exclude from forwarding. + except: [] + # Use TCP for forwarding, even if the original request was UDP. + force_tcp: false + # Prefer UDP for forwarding; fallback to TCP if the response is truncated. + prefer_udp: false + # Number of consecutive failed health checks before marking an upstream as down. +# max_fails: 2 + # Time after which cached connections expire. +# expire: 10s + # TLS properties for secure connections can be set here. +# tls: +# cert_file: "" +# key_file: "" +# ca_file: "" +# tls_servername: "" + # Policy for selecting upstream servers: random (default), round_robin, sequential. +# policy: "random" + # Health check configuration for upstream servers. +# health_check: "" + # Maximum number of concurrent DNS queries allowed. + max_concurrent: 1000 \ No newline at end of file diff --git a/builtin/core/roles/defaults/defaults/main/05-storage_class.yaml b/builtin/core/roles/defaults/defaults/main/05-storage_class.yaml new file mode 100644 index 000000000..7d0726d13 --- /dev/null +++ b/builtin/core/roles/defaults/defaults/main/05-storage_class.yaml @@ -0,0 +1,32 @@ +# Storage class configuration for Kubernetes persistent storage integration +storage_class: + # Local storage class configuration + local: + enabled: true # Enable local storage class + default: true # Set as the default storage class + provisioner_image: + registry: >- + {{ .image_registry.dockerio_registry }} + repository: openebs/provisioner-localpv + tag: 4.3.0 + linux_utils_image: + registry: >- + {{ .image_registry.dockerio_registry }} + repository: openebs/linux-utils + tag: 4.3.0 + path: /var/openebs/local # Host path for local storage volumes + + # NFS storage class configuration + nfs: + # Ensure nfs-utils is installed on every node in the k8s_cluster group + enabled: false # Enable NFS storage class + default: false # Set as the default storage class + # NFS server address + server: >- + {{ .groups.nfs | default list | first }} + path: /share/kubernetes # NFS export path for persistent volumes + + # ========== storageclass ========== + # ========== storageclass: nfs ========== + # nfs provisioner helm version + nfs_provisioner_version: 4.0.20 \ No newline at end of file diff --git a/builtin/core/roles/defaults/tasks/main.yaml b/builtin/core/roles/defaults/tasks/main.yaml new file mode 100644 index 000000000..6c06e004b --- /dev/null +++ b/builtin/core/roles/defaults/tasks/main.yaml @@ -0,0 +1,79 @@ +- name: Defaults | Load defaults based on Kubernetes version + block: + - name: Defaults | Load version-specific settings for Kubernetes + tags: ["image_registry"] + when: .kubernetes.kube_version | empty | not + include_vars: >- + {{ slice (.kubernetes.kube_version | splitList ".") 0 2 | join "." }}.yaml + - name: Defaults | Load architecture-specific download URLs for each artifact version + include_vars: 10-download.yaml + +- name: Defaults | Reset temporary directory + command: | + if [ -d {{ .tmp_dir }} ]; then + rm -rf {{ .tmp_dir }} + fi + mkdir -m 777 -p {{ .tmp_dir }} + +- name: Defaults | Determine operating system architecture for each node + set_fact: + binary_type: >- + {{- if .transform_architectures.amd64 | has .os.architecture -}} + amd64 + {{- else if .transform_architectures.arm64 | has .os.architecture -}} + arm64 + {{- end -}} + +- name: Defaults | Gather Kubernetes service status + when: .groups.k8s_cluster | has .inventory_hostname + block: + - name: Defaults | Get kubelet.service LoadState + command: systemctl show kubelet.service -p LoadState --value + register: kubernetes_install_LoadState + - name: Defaults | Get kubelet.service ActiveState + command: systemctl show kubelet.service -p ActiveState --value + register: kubernetes_install_ActiveState + - name: Defaults | Get installed Kubernetes version + ignore_errors: true + command: kubelet --version + register: kubernetes_install_version + +- name: Defaults | Gather ETCD service status + when: .groups.etcd | has .inventory_hostname + block: + - name: Defaults | Get etcd.service LoadState and save to variable + command: systemctl show etcd.service -p LoadState --value + register: etcd_install_LoadState + - name: Defaults | Get etcd.service ActiveState and save to variable + command: systemctl show etcd.service -p ActiveState --value + register: etcd_install_ActiveState + - name: Defaults | Get installed etcd version + ignore_errors: true + command: etcd --version + register: etcd_install_version + register_type: yaml + +- name: Defaults | Select the initialization node for the cluster + run_once: true + add_hostvars: + hosts: k8s_cluster + vars: + init_kubernetes_node: >- + {{- $initNodes := list -}} + {{- $notInitNodes := list -}} + {{- range .groups.kube_control_plane -}} + {{- if index $.hostvars . "kubernetes_install_LoadState" "stdout" | eq "loaded" -}} + {{- $initNodes = append $initNodes . -}} + {{- else if index $.hostvars . "kubernetes_install_LoadState" "stdout" | eq "not-found" -}} + {{- $notInitNodes = append $notInitNodes . -}} + {{- end -}} + {{- end -}} + {{- if $initNodes | len | eq 1 -}} + {{ $initNodes | first }} + {{- else if $initNodes | len | lt 1 -}} + {{ index $initNodes (randInt 0 ((sub ($initNodes | len) 1) | int)) }} + {{- else if $notInitNodes | len | eq 1 -}} + {{ $notInitNodes | first }} + {{- else if $notInitNodes | len | lt 1 -}} + {{ index $notInitNodes (randInt 0 ((sub ($notInitNodes | len) 1) | int)) }} + {{- end -}} \ No newline at end of file diff --git a/builtin/core/roles/defaults/vars/10-download.yaml b/builtin/core/roles/defaults/vars/10-download.yaml new file mode 100644 index 000000000..05b48fcdf --- /dev/null +++ b/builtin/core/roles/defaults/vars/10-download.yaml @@ -0,0 +1,224 @@ +download: + # if set as "cn", so that online downloads will try to use available domestic sources whenever possible. + zone: "" + arch: [ "amd64" ] + # offline artifact package for kk. + artifact_file: "" + # the md5_file of artifact_file. + artifact_md5: "" + artifact_url: + etcd: + amd64: >- + {{- if .download.zone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/etcd/release/download/{{ .etcd.etcd_version }}/etcd-{{ .etcd.etcd_version }}-linux-amd64.tar.gz + {{- else -}} + https://github.com/etcd-io/etcd/releases/download/{{ .etcd.etcd_version }}/etcd-{{ .etcd.etcd_version }}-linux-amd64.tar.gz + {{- end -}} + arm64: >- + {{- if .download.zone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/etcd/release/download/{{ .etcd.etcd_version }}/etcd-{{ .etcd.etcd_version }}-linux-arm64.tar.gz + {{- else -}} + https://github.com/etcd-io/etcd/releases/download/{{ .etcd.etcd_version }}/etcd-{{ .etcd.etcd_version }}-linux-arm64.tar.gz + {{- end -}} + kubeadm: + amd64: >- + {{- if .download.zone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/release/{{ .kubernetes.kube_version }}/bin/linux/amd64/kubeadm + {{- else -}} + https://dl.k8s.io/release/{{ .kubernetes.kube_version }}/bin/linux/amd64/kubeadm + {{- end -}} + arm64: >- + {{- if .download.zone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/release/{{ .kubernetes.kube_version }}/bin/linux/arm64/kubeadm + {{- else -}} + https://dl.k8s.io/release/{{ .kubernetes.kube_version }}/bin/linux/arm64/kubeadm + {{- end -}} + kubelet: + amd64: >- + {{- if .download.zone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/release/{{ .kubernetes.kube_version }}/bin/linux/amd64/kubelet + {{- else -}} + https://dl.k8s.io/release/{{ .kubernetes.kube_version }}/bin/linux/amd64/kubelet + {{- end -}} + arm64: >- + {{- if .download.zone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/release/{{ .kubernetes.kube_version }}/bin/linux/arm64/kubelet + {{- else -}} + https://dl.k8s.io/release/{{ .kubernetes.kube_version }}/bin/linux/arm64/kubelet + {{- end -}} + kubectl: + amd64: >- + {{- if .download.zone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/release/{{ .kubernetes.kube_version }}/bin/linux/amd64/kubectl + {{- else -}} + https://dl.k8s.io/release/{{ .kubernetes.kube_version }}/bin/linux/amd64/kubectl + {{- end -}} + arm64: >- + {{- if .download.zone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/release/{{ .kubernetes.kube_version }}/bin/linux/arm64/kubectl + {{- else -}} + https://dl.k8s.io/release/{{ .kubernetes.kube_version }}/bin/linux/arm64/kubectl + {{- end -}} + cni_plugins: + amd64: >- + {{- if .download.zone | eq "cn" -}} + https://github.com/containernetworking/plugins/releases/download/{{ .cni.cni_plugins_version }}/cni-plugins-linux-amd64-{{ .cni.cni_plugins_version }}.tgz + {{- else -}} + https://containernetworking.pek3b.qingstor.com/plugins/releases/download/{{ .cni.cni_plugins_version }}/cni-plugins-linux-amd64-{{ .cni.cni_plugins_version }}.tgz + {{- end -}} + arm64: >- + {{- if .download.zone | eq "cn" -}} + https://github.com/containernetworking/plugins/releases/download/{{ .cni.cni_plugins_version }}/cni-plugins-linux-arm64-{{ .cni.cni_plugins_version }}.tgz + {{- else -}} + https://containernetworking.pek3b.qingstor.com/plugins/releases/download/{{ .cni.cni_plugins_version }}/cni-plugins-linux-arm64-{{ .cni.cni_plugins_version }}.tgz + {{- end -}} + helm: + amd64: >- + {{- if .download.zone | eq "cn" -}} + https://kubernetes-helm.pek3b.qingstor.com/helm-{{ .kubernetes.helm_version }}-linux-amd64.tar.gz + {{- else -}} + https://get.helm.sh/helm-{{ .kubernetes.helm_version }}-linux-amd64.tar.gz + {{- end -}} + arm64: >- + {{- if .download.zone | eq "cn" -}} + https://kubernetes-helm.pek3b.qingstor.com/helm-{{ .kubernetes.helm_version }}-linux-arm64.tar.gz + {{- else -}} + https://get.helm.sh/helm-{{ .kubernetes.helm_version }}-linux-arm64.tar.gz + {{- end -}} + crictl: + amd64: >- + {{- if .download.zone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/cri-tools/releases/download/{{ .cri.crictl_version }}/crictl-{{ .cri.crictl_version }}-linux-amd64.tar.gz + {{- else -}} + https://github.com/kubernetes-sigs/cri-tools/releases/download/{{ .cri.crictl_version }}/crictl-{{ .cri.crictl_version }}-linux-amd64.tar.gz + {{- end -}} + arm64: >- + {{- if .download.zone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/cri-tools/releases/download/{{ .cri.crictl_version }}/crictl-{{ .cri.crictl_version }}-linux-arm64.tar.gz + {{- else -}} + https://github.com/kubernetes-sigs/cri-tools/releases/download/{{ .cri.crictl_version }}/crictl-{{ .cri.crictl_version }}-linux-arm64.tar.gz + {{- end -}} + docker: + amd64: >- + {{- if .download.zone | eq "cn" -}} + https://mirrors.aliyun.com/docker-ce/linux/static/stable/x86_64/docker-{{ .cri.docker_version }}.tgz + {{- else -}} + https://download.docker.com/linux/static/stable/x86_64/docker-{{ .cri.docker_version }}.tgz + {{- end -}} + arm64: >- + {{- if .download.zone | eq "cn" -}} + https://mirrors.aliyun.com/docker-ce/linux/static/stable/aarch64/docker-{{ .cri.docker_version }}.tgz + {{- else -}} + https://download.docker.com/linux/static/stable/aarch64/docker-{{ .cri.docker_version }}.tgz + {{- end -}} + cridockerd: + amd64: >- + {{- if .download.zone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/releases/download/{{ .cri.cridockerd_version }}/cri-dockerd-{{ .cri.cridockerd_version | default "" | trimPrefix "v" }}.amd64.tgz + {{- else -}} + https://github.com/Mirantis/cri-dockerd/releases/download/{{ .cri.cridockerd_version }}/cri-dockerd-{{ .cri.cridockerd_version | default "" | trimPrefix "v" }}.amd64.tgz + {{- end -}} + arm64: >- + {{- if .download.zone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/releases/download/{{ .cri.cridockerd_version }}/cri-dockerd-{{ .cri.cridockerd_version | default "" | trimPrefix "v" }}.arm64.tgz + {{- else -}} + https://github.com/Mirantis/cri-dockerd/releases/download/{{ .cri.cridockerd_version }}/cri-dockerd-{{ .cri.cridockerd_version | default "" | trimPrefix "v" }}.arm64.tgz + {{- end -}} + containerd: + amd64: >- + {{- if .download.zone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/containerd/containerd/releases/download/{{ .cri.containerd_version }}/containerd-{{ .cri.containerd_version | default "" | trimPrefix "v" }}-linux-amd64.tar.gz + {{- else -}} + https://github.com/containerd/containerd/releases/download/{{ .cri.containerd_version }}/containerd-{{ .cri.containerd_version | default "" | trimPrefix "v" }}-linux-amd64.tar.gz + {{- end -}} + arm64: >- + {{- if .download.zone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/containerd/containerd/releases/download/{{ .cri.containerd_version }}/containerd-{{ .cri.containerd_version | default "" | trimPrefix "v" }}-linux-arm64.tar.gz + {{- else -}} + https://github.com/containerd/containerd/releases/download/{{ .cri.containerd_version }}/containerd-{{ .cri.containerd_version | default "" | trimPrefix "v" }}-linux-arm64.tar.gz + {{- end -}} + runc: + amd64: >- + {{- if .download.zone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/opencontainers/runc/releases/download/{{ .cri.runc_version }}/runc.amd64 + {{- else -}} + https://github.com/opencontainers/runc/releases/download/{{ .cri.runc_version }}/runc.amd64 + {{- end -}} + arm64: >- + {{- if .download.zone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/opencontainers/runc/releases/download/{{ .cri.runc_version }}/runc.arm64 + {{- else -}} + https://github.com/opencontainers/runc/releases/download/{{ .cri.runc_version }}/runc.arm64 + {{- end -}} + dockercompose: + amd64: >- + {{- if .download.zone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/docker/compose/releases/download/{{ .cri.dockercompose_version }}/docker-compose-linux-x86_64 + {{- else -}} + https://github.com/docker/compose/releases/download/{{ .cri.dockercompose_version }}/docker-compose-linux-x86_64 + {{- end -}} + arm64: >- + {{- if .download.zone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/docker/compose/releases/download/{{ .cri.dockercompose_version }}/docker-compose-linux-aarch64 + {{- else -}} + https://github.com/docker/compose/releases/download/{{ .cri.dockercompose_version }}/docker-compose-linux-aarch64 + {{- end -}} +# docker_registry: +# amd64: >- +# {{- if .download.zone | eq "cn" -}} +# https://kubernetes-release.pek3b.qingstor.com/registry/{{ .image_registry.docker_registry_version }}/docker-registry-{{ .image_registry.docker_registry_version }}-linux-amd64.tgz +# {{- else -}} +# https://github.com/kubesphere/kubekey/releases/download/{{ .image_registry.docker_registry_version }}/docker-registry-{{ .image_registry.docker_registry_version }}-linux-amd64.tgz +# {{- end -}} +# arm64: >- +# {{- if .download.zone | eq "cn" -}} +# https://kubernetes-release.pek3b.qingstor.com/registry/{{ .image_registry.docker_registry_version }}/docker-registry-{{ .image_registry.docker_registry_version }}-linux-arm64.tgz +# {{- else -}} +# https://github.com/kubesphere/kubekey/releases/download/{{ .image_registry.docker_registry_version }}/docker-registry-{{ .image_registry.docker_registry_version }}-linux-arm64.tgz +# {{- end -}} + harbor: + amd64: >- + {{- if .download.zone | eq "cn" -}} + https://github.com/goharbor/harbor/releases/download/{{ .image_registry.harbor_version }}/harbor-offline-installer-{{ .image_registry.harbor_version }}.tgz + {{- else -}} + https://github.com/goharbor/harbor/releases/download/{{ .image_registry.harbor_version }}/harbor-offline-installer-{{ .image_registry.harbor_version }}.tgz + {{- end -}} +# arm64: >- +# {{- if .download.zone | eq "cn" -}} +# https://github.com/goharbor/harbor/releases/download/{{ .image_registry.harbor_version }}/harbor-{{ .image_registry.harbor_version }}-linux-arm64.tgz +# {{- else -}} +# https://github.com/goharbor/harbor/releases/download/{{ .image_registry.harbor_version }}/harbor-{{ .image_registry.harbor_version }}-linux-arm64.tgz +# {{- end -}} +# keepalived: +# amd64: >- +# {{- if .download.zone | eq "cn" -}} +# https://kubernetes-release.pek3b.qingstor.com/osixia/keepalived/releases/download/{{ .image_registry.keepalived_version }}/keepalived-{{ .image_registry.keepalived_version }}-linux-amd64.tgz +# {{- else -}} +# https://github.com/osixia/keepalived/releases/download/{{ .image_registry.keepalived_version }}/keepalived-{{ .image_registry.keepalived_version }}-linux-amd64.tgz +# {{- end -}} +# arm64: >- +# {{- if .download.zone | eq "cn" -}} +# https://kubernetes-release.pek3b.qingstor.com/osixia/keepalived/releases/download/{{ .image_registry.keepalived_version }}/keepalived-{{ .image_registry.keepalived_version }}-linux-arm64.tgz +# {{- else -}} +# https://github.com/osixia/keepalived/releases/download/{{ .image_registry.keepalived_version }}/keepalived-{{ .image_registry.keepalived_version }}-linux-arm64.tgz +# {{- end -}} + # Notice: In the early calico helm chart, appVersion is not same as version(eg. v3.17.4) + calico: https://github.com/projectcalico/calico/releases/download/{{ .cni.calico_version }}/tigera-operator-{{ .cni.calico_version }}.tgz + calicoctl: + amd64: >- + {{- if .download.zone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/projectcalico/calico/releases/download/{{ .cni.calico_version }}/calicoctl-linux-amd64 + {{- else -}} + https://github.com/projectcalico/calico/releases/download/{{ .cni.calico_version }}/calicoctl-linux-amd64 + {{- end -}} + arm64: >- + {{- if .download.zone | eq "cn" -}} + https://kubernetes-release.pek3b.qingstor.com/projectcalico/calico/releases/download/{{ .cni.calico_version }}/calicoctl-linux-arm64 + {{- else -}} + https://github.com/projectcalico/calico/releases/download/{{ .cni.calico_version }}/calicoctl-linux-arm64 + {{- end -}} + cilium: https://helm.cilium.io/cilium-{{ .cni.cilium_version }}.tgz + kubeovn: https://kubeovn.github.io/kube-ovn/kube-ovn-{{ .cni.kubeovn_version }}.tgz + hybridnet: https://github.com/alibaba/hybridnet/releases/download/helm-chart-{{ .cni.hybridnet_version }}/hybridnet-{{ .cni.hybridnet_version }}.tgz + nfs_provisioner: https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner/releases/download/nfs-subdir-external-provisioner-4.0.18/nfs-subdir-external-provisioner-{{ .storage_class.nfs_provisioner_version }}.tgz + download_image: true diff --git a/builtin/core/roles/defaults/vars/v1.23.yaml b/builtin/core/roles/defaults/vars/v1.23.yaml new file mode 100644 index 000000000..f69e4c313 --- /dev/null +++ b/builtin/core/roles/defaults/vars/v1.23.yaml @@ -0,0 +1,96 @@ +kubernetes: + # helm binary + helm_version: v3.8.2 +etcd: + # etcd binary + etcd_version: v3.5.4 +image_registry: + # keepalived image tag. Used for load balancing when there are multiple image registry nodes. + keepalived_version: 2.0.20 + # ========== image registry: harbor ========== + # harbor image tag + harbor_version: v2.6.3 + # docker-compose binary + dockercompose_version: v2.12.2 + # ========== image registry: docker-registry ========== + # docker-registry image tag + docker_registry_version: 2.8.3 +cri: + # support: containerd,docker + container_manager: docker + sandbox_image: + tag: "3.6" + # ========== cri ========== + # crictl binary + crictl_version: v1.23.0 + # ========== cri: docker ========== + # docker binary + docker_version: 20.10.18 + # cridockerd. Required when kube_version is greater than 1.24 + cridockerd_version: v0.3.10 + # ========== cri: containerd ========== + # containerd binary + containerd_version: v1.6.8 + # runc binary + runc_version: v1.1.4 +cni: + multus: + image: + tag: v3.9.3 + # ========== cni ========== + # cni_plugins binary (optional) + # cni_plugins_version: v1.1.1 + # ========== cni: calico ========== + # calicoctl binary + calico_version: v3.24.5 + # ========== cni: cilium ========== + # cilium helm + cilium_version: 1.12.6 + # ========== cni: kubeovn ========== + # kubeovn helm + kubeovn_version: 1.10.0 + # ========== cni: hybridnet ========== + # hybridnet helm + hybridnet_version: 0.6.8 +storage_class: + # ========== storageclass ========== + # ========== storageclass: local ========== + local: + provisioner_image: + tag: 3.3.0 + linux_utils_image: + tag: 3.3.0 + # ========== storageclass: nfs ========== + # nfs provisioner helm version + nfs_provisioner_version: 4.0.2 +dns: + dns_image: + tag: v1.8.6 + dns_cache_image: + tag: 1.21.1 + image_manifests: + - docker.io/calico/apiserver:v3.24.5 + - docker.io/calico/cni:v3.24.5 + - docker.io/calico/csi:v3.24.5 + - docker.io/calico/kube-controllers:v3.24.5 + - docker.io/calico/node-driver-registrar:v3.24.5 + - docker.io/calico/node:v3.24.5 + - docker.io/calico/pod2daemon-flexvol:v3.24.5 + - docker.io/kubesphere/k8s-dns-node-cache:1.21.1 + - docker.io/openebs/provisioner-localpv:3.3.0 + - docker.io/kubesphere/coredns:v1.8.6 + - docker.io/kubesphere/kube-apiserver:{{ .kube_version }} + - docker.io/kubesphere/kube-controller-manager:{{ .kube_version }} + - docker.io/kubesphere/kube-proxy:{{ .kube_version }} + - docker.io/kubesphere/kube-scheduler:{{ .kube_version }} + - docker.io/kubesphere/pause:3.6 + - quay.io/tigera/operator:v1.28.5 + - docker.io/calico/ctl:v3.24.5 + - docker.io/calico/typha:v3.24.5 + - docker.io/calico/apiserver:v3.24.5 + - docker.io/calico/kube-controllers:v3.24.5 + - docker.io/calico/node:v3.24.5 + - docker.io/calico/pod2daemon-flexvol:v3.24.5 + - docker.io/calico/cni:v3.24.5 + - docker.io/calico/node-driver-registrar:v3.24.5 + - docker.io/calico/csi:v3.24.5 diff --git a/builtin/core/roles/defaults/vars/v1.24.yaml b/builtin/core/roles/defaults/vars/v1.24.yaml new file mode 100644 index 000000000..770e65332 --- /dev/null +++ b/builtin/core/roles/defaults/vars/v1.24.yaml @@ -0,0 +1,97 @@ +kubernetes: + # helm binary + helm_version: v3.10.3 +etcd: + # etcd binary + etcd_version: v3.5.6 +image_registry: + # ========== image registry ========== + # keepalived image tag. Used for load balancing when there are multiple image registry nodes. + keepalived_version: v2.0.20 + # ========== image registry: harbor ========== + # harbor image tag + harbor_version: v2.7.1 + # docker-compose binary + dockercompose_version: v2.14.0 + # ========== image registry: docker-registry ========== + # docker-registry image tag + docker_registry_version: 2.8.3 +cri: + # support: containerd,docker + container_manager: containerd + sandbox_image: + tag: "3.6" + # ========== cri ========== + # crictl binary + crictl_version: v1.24.0 + # ========== cri: docker ========== + # docker binary + docker_version: 20.10.24 + # cridockerd. Required when kube_version is greater than 1.24 + cridockerd_version: v0.3.1 + # ========== cri: containerd ========== + # containerd binary + containerd_version: v1.6.16 + # runc binary + runc_version: v1.1.4 +cni: + multus: + image: + tag: v3.10.1 + # ========== cni ========== + # cni_plugins binary (optional) + # cni_plugins_version: v1.1.1 + # ========== cni: calico ========== + # calicoctl binary + calico_version: v3.25.1 + # ========== cni: cilium ========== + # cilium helm + cilium_version: 1.13.5 + # ========== cni: kubeovn ========== + # kubeovn helm + kubeovn_version: 1.10.0 + # ========== cni: hybridnet ========== + # hybridnet helm + hybridnet_version: 0.6.8 +storage_class: + # ========== storageclass ========== + # ========== storageclass: local ========== + local: + provisioner_image: + tag: 3.4.0 + linux_utils_image: + tag: 3.4.0 + # ========== storageclass: nfs ========== + # nfs provisioner helm version + nfs_provisioner_version: 4.0.2 + dns: + dns_image: + tag: v1.8.6 + dns_cache_image: + tag: 1.22.20 + image_manifests: + - docker.io/calico/apiserver:v3.25.1 + - docker.io/calico/cni:v3.25.1 + - docker.io/calico/csi:v3.25.1 + - docker.io/calico/kube-controllers:v3.25.1 + - docker.io/calico/node-driver-registrar:v3.25.1 + - docker.io/calico/node:v3.25.1 + - docker.io/calico/pod2daemon-flexvol:v3.25.1 + - docker.io/kubesphere/k8s-dns-node-cache:1.22.20 + - docker.io/openebs/provisioner-localpv:3.4.0 + - docker.io/kubesphere/coredns:v1.8.6 + - docker.io/kubesphere/kube-apiserver:{{ .kube_version }} + - docker.io/kubesphere/kube-controller-manager:{{ .kube_version }} + - docker.io/kubesphere/kube-proxy:{{ .kube_version }} + - docker.io/kubesphere/kube-scheduler:{{ .kube_version }} + - docker.io/kubesphere/pause:3.6 + - quay.io/tigera/operator:v1.29.3 + - docker.io/calico/ctl:v3.25.1 + - docker.io/calico/typha:v3.25.1 + - docker.io/calico/apiserver:v3.25.1 + - docker.io/calico/kube-controllers:v3.25.1 + - docker.io/calico/node:v3.25.1 + - docker.io/calico/pod2daemon-flexvol:v3.25.1 + - docker.io/calico/cni:v3.25.1 + - docker.io/calico/node-driver-registrar:v3.25.1 + - docker.io/calico/csi:v3.25.1 diff --git a/builtin/core/roles/defaults/vars/v1.25.yaml b/builtin/core/roles/defaults/vars/v1.25.yaml new file mode 100644 index 000000000..7c37c3b73 --- /dev/null +++ b/builtin/core/roles/defaults/vars/v1.25.yaml @@ -0,0 +1,97 @@ +kubernetes: + # helm binary + helm_version: v3.10.3 +etcd: + # etcd binary + etcd_version: v3.5.7 +image_registry: + # ========== image registry ========== + # keepalived image tag. Used for load balancing when there are multiple image registry nodes. + keepalived_version: 2.0.20 + # ========== image registry: harbor ========== + # harbor image tag + harbor_version: v2.8.1 + # docker-compose binary + dockercompose_version: v2.15.1 + # ========== image registry: docker-registry ========== + # docker-registry image tag + docker_registry_version: 2.8.3 +cri: + # support: containerd,docker + container_manager: containerd + sandbox_image: + tag: "3.6" + # ========== cri ========== + # crictl binary + crictl_version: v1.25.0 + # ========== cri: docker ========== + # docker binary + docker_version: 20.10.24 + # cridockerd. Required when kube_version is greater than 1.24 + cridockerd_version: v0.3.1 + # ========== cri: containerd ========== + # containerd binary + containerd_version: v1.6.19 + # runc binary + runc_version: v1.1.4 +cni: + multus: + image: + tag: v3.11.3 + # ========== cni ========== + # cni_plugins binary (optional) + # cni_plugins_version: v1.1.1 + # ========== cni: calico ========== + # calicoctl binary + calico_version: v3.25.1 + # ========== cni: cilium ========== + # cilium helm + cilium_version: 1.13.5 + # ========== cni: kubeovn ========== + # kubeovn helm + kubeovn_version: 1.10.0 + # ========== cni: hybridnet ========== + # hybridnet helm + hybridnet_version: 0.6.8 +storage_class: + # ========== storageclass ========== + # ========== storageclass: local ========== + local: + provisioner_image: + tag: 3.4.0 + linux_utils_image: + tag: 3.4.0 + # ========== storageclass: nfs ========== + # nfs provisioner helm version + nfs_provisioner_version: 4.0.8 +dns: + dns_image: + tag: v1.9.3 + dns_cache_image: + tag: 1.22.20 + image_manifests: + - docker.io/calico/apiserver:v3.25.1 + - docker.io/calico/cni:v3.25.1 + - docker.io/calico/csi:v3.25.1 + - docker.io/calico/kube-controllers:v3.25.1 + - docker.io/calico/node-driver-registrar:v3.25.1 + - docker.io/calico/node:v3.25.1 + - docker.io/calico/pod2daemon-flexvol:v3.25.1 + - docker.io/kubesphere/k8s-dns-node-cache:1.22.20 + - docker.io/openebs/provisioner-localpv:3.4.0 + - docker.io/kubesphere/coredns:v1.9.3 + - docker.io/kubesphere/kube-apiserver:{{ .kube_version }} + - docker.io/kubesphere/kube-controller-manager:{{ .kube_version }} + - docker.io/kubesphere/kube-proxy:{{ .kube_version }} + - docker.io/kubesphere/kube-scheduler:{{ .kube_version }} + - docker.io/kubesphere/pause:3.6 + - quay.io/tigera/operator:v1.29.3 + - docker.io/calico/ctl:v3.25.1 + - docker.io/calico/typha:v3.25.1 + - docker.io/calico/apiserver:v3.25.1 + - docker.io/calico/kube-controllers:v3.25.1 + - docker.io/calico/node:v3.25.1 + - docker.io/calico/pod2daemon-flexvol:v3.25.1 + - docker.io/calico/cni:v3.25.1 + - docker.io/calico/node-driver-registrar:v3.25.1 + - docker.io/calico/csi:v3.25.1 diff --git a/builtin/core/roles/defaults/vars/v1.26.yaml b/builtin/core/roles/defaults/vars/v1.26.yaml new file mode 100644 index 000000000..2af186a6f --- /dev/null +++ b/builtin/core/roles/defaults/vars/v1.26.yaml @@ -0,0 +1,97 @@ +kubernetes: + # helm binary + helm_version: v3.11.2 +etcd: + # etcd binary + etcd_version: v3.5.8 +image_registry: + # ========== image registry ========== + # keepalived image tag. Used for load balancing when there are multiple image registry nodes. + keepalived_version: 2.0.20 + # ========== image registry: harbor ========== + # harbor image tag + harbor_version: v2.9.1 + # docker-compose binary + dockercompose_version: v2.16.0 + # ========== image registry: docker-registry ========== + # docker-registry image tag + docker_registry_version: 2.8.3 +cri: + # support: containerd,docker + container_manager: containerd + sandbox_image: + tag: "3.7" + # ========== cri ========== + # crictl binary + crictl_version: v1.26.0 + # ========== cri: docker ========== + # docker binary + docker_version: 23.0.6 + # cridockerd. Required when kube_version is greater than 1.24 + cridockerd_version: v0.3.1 + # ========== cri: containerd ========== + # containerd binary + containerd_version: v1.6.21 + # runc binary + runc_version: v1.1.5 +cni: + multus: + image: + tag: v4.0.2 + # ========== cni ========== + # cni_plugins binary (optional) + # cni_plugins_version: v1.2.0 + # ========== cni: calico ========== + # calicoctl binary + calico_version: v3.26.1 + # ========== cni: cilium ========== + # cilium helm + cilium_version: 1.13.5 + # ========== cni: kubeovn ========== + # kubeovn helm + kubeovn_version: 1.10.0 + # ========== cni: hybridnet ========== + # hybridnet helm + hybridnet_version: 0.6.8 +storage_class: + # ========== storageclass ========== + # ========== storageclass: local ========== + local: + provisioner_image: + tag: 3.4.0 + linux_utils_image: + tag: 3.4.0 + # ========== storageclass: nfs ========== + # nfs provisioner helm version + nfs_provisioner_version: 4.0.8 +dns: + dns_image: + tag: v1.9.3 + dns_cache_image: + tag: 1.22.20 + image_manifests: + - docker.io/calico/apiserver:v3.26.1 + - docker.io/calico/cni:v3.26.1 + - docker.io/calico/csi:v3.26.1 + - docker.io/calico/kube-controllers:v3.26.1 + - docker.io/calico/node-driver-registrar:v3.26.1 + - docker.io/calico/node:v3.26.1 + - docker.io/calico/pod2daemon-flexvol:v3.26.1 + - docker.io/kubesphere/k8s-dns-node-cache:1.22.20 + - docker.io/openebs/provisioner-localpv:3.4.0 + - docker.io/kubesphere/coredns:v1.9.3 + - docker.io/kubesphere/kube-apiserver:{{ .kube_version }} + - docker.io/kubesphere/kube-controller-manager:{{ .kube_version }} + - docker.io/kubesphere/kube-proxy:{{ .kube_version }} + - docker.io/kubesphere/kube-scheduler:{{ .kube_version }} + - docker.io/kubesphere/pause:3.7 + - quay.io/tigera/operator:v1.30.4 + - docker.io/calico/ctl:v3.26.1 + - docker.io/calico/typha:v3.26.1 + - docker.io/calico/apiserver:v3.26.1 + - docker.io/calico/kube-controllers:v3.26.1 + - docker.io/calico/node:v3.26.1 + - docker.io/calico/pod2daemon-flexvol:v3.26.1 + - docker.io/calico/cni:v3.26.1 + - docker.io/calico/node-driver-registrar:v3.26.1 + - docker.io/calico/csi:v3.26.1 diff --git a/builtin/core/roles/defaults/vars/v1.27.yaml b/builtin/core/roles/defaults/vars/v1.27.yaml new file mode 100644 index 000000000..d0a23a504 --- /dev/null +++ b/builtin/core/roles/defaults/vars/v1.27.yaml @@ -0,0 +1,97 @@ +kubernetes: + # helm binary + helm_version: v3.12.1 +etcd: + # etcd binary + etcd_version: v3.5.9 +image_registry: + # ========== image registry ========== + # keepalived image tag. Used for load balancing when there are multiple image registry nodes. + keepalived_version: 2.0.20 + # ========== image registry: harbor ========== + # harbor image tag + harbor_version: v2.10.1 + # docker-compose binary + dockercompose_version: v2.20.3 + # ========== image registry: docker-registry ========== + # docker-registry image tag + docker_registry_version: 2.8.3 +cri: + # support: containerd,docker + container_manager: containerd + sandbox_image: + tag: "3.7" + # ========== cri ========== + # crictl binary + crictl_version: v1.27.0 + # ========== cri: docker ========== + # docker binary + docker_version: 23.0.6 + # cridockerd. Required when kube_version is greater than 1.24 + cridockerd_version: v0.3.1 + # ========== cri: containerd ========== + # containerd binary + containerd_version: v1.7.2 + # runc binary + runc_version: v1.1.7 +cni: + multus: + image: + tag: v4.0.2 + # ========== cni ========== + # cni_plugins binary (optional) + # cni_plugins_version: v1.2.0 + # ========== cni: calico ========== + # calicoctl binary + calico_version: v3.26.1 + # ========== cni: cilium ========== + # cilium helm + cilium_version: 1.14.2 + # ========== cni: kubeovn ========== + # kubeovn helm + kubeovn_version: 1.11.0 + # ========== cni: hybridnet ========== + # hybridnet helm + hybridnet_version: 0.6.8 +storage_class: + # ========== storageclass ========== + # ========== storageclass: local ========== + local: + provisioner_image: + tag: 3.4.0 + linux_utils_image: + tag: 3.4.0 + # ========== storageclass: nfs ========== + # nfs provisioner helm version + nfs_provisioner_version: 4.0.10 +dns: + dns_image: + tag: v1.10.1 + dns_cache_image: + tag: 1.22.20 + image_manifests: + - docker.io/calico/apiserver:v3.26.1 + - docker.io/calico/cni:v3.26.1 + - docker.io/calico/csi:v3.26.1 + - docker.io/calico/kube-controllers:v3.26.1 + - docker.io/calico/node-driver-registrar:v3.26.1 + - docker.io/calico/node:v3.26.1 + - docker.io/calico/pod2daemon-flexvol:v3.26.1 + - docker.io/kubesphere/k8s-dns-node-cache:1.22.20 + - docker.io/openebs/provisioner-localpv:3.4.0 + - docker.io/kubesphere/coredns:v1.10.1 + - docker.io/kubesphere/kube-apiserver:{{ .kube_version }} + - docker.io/kubesphere/kube-controller-manager:{{ .kube_version }} + - docker.io/kubesphere/kube-proxy:{{ .kube_version }} + - docker.io/kubesphere/kube-scheduler:{{ .kube_version }} + - docker.io/kubesphere/pause:3.7 + - quay.io/tigera/operator:v1.30.4 + - docker.io/calico/ctl:v3.26.1 + - docker.io/calico/typha:v3.26.1 + - docker.io/calico/apiserver:v3.26.1 + - docker.io/calico/kube-controllers:v3.26.1 + - docker.io/calico/node:v3.26.1 + - docker.io/calico/pod2daemon-flexvol:v3.26.1 + - docker.io/calico/cni:v3.26.1 + - docker.io/calico/node-driver-registrar:v3.26.1 + - docker.io/calico/csi:v3.26.1 diff --git a/builtin/core/roles/defaults/vars/v1.28.yaml b/builtin/core/roles/defaults/vars/v1.28.yaml new file mode 100644 index 000000000..7683f58fe --- /dev/null +++ b/builtin/core/roles/defaults/vars/v1.28.yaml @@ -0,0 +1,97 @@ +kubernetes: + # helm binary + helm_version: v3.12.1 +etcd: + # etcd binary + etcd_version: v3.5.9 +image_registry: + # ========== image registry ========== + # keepalived image tag. Used for load balancing when there are multiple image registry nodes. + keepalived_version: 2.0.20 + # ========== image registry: harbor ========== + # harbor image tag + harbor_version: v2.10.1 + # docker-compose binary + dockercompose_version: v2.20.3 + # ========== image registry: docker-registry ========== + # docker-registry image tag + docker_registry_version: 2.8.3 +cri: + # support: containerd,docker + container_manager: containerd + sandbox_image: + tag: "3.8" + # ========== cri ========== + # crictl binary + crictl_version: v1.28.0 + # ========== cri: docker ========== + # docker binary + docker_version: 24.0.6 + # cridockerd. Required when kube_version is greater than 1.24 + cridockerd_version: v0.3.1 + # ========== cri: containerd ========== + # containerd binary + containerd_version: v1.7.3 + # runc binary + runc_version: v1.1.7 +cni: + multus: + image: + tag: v4.1.0 + # ========== cni ========== + # cni_plugins binary (optional) + # cni_plugins_version: v1.2.0 + # ========== cni: calico ========== + # calicoctl binary + calico_version: v3.28.2 + # ========== cni: cilium ========== + # cilium helm + cilium_version: 1.15.0 + # ========== cni: kubeovn ========== + # kubeovn helm + kubeovn_version: 1.12.0 + # ========== cni: hybridnet ========== + # hybridnet helm + hybridnet_version: 0.6.8 +storage_class: + # ========== storageclass ========== + # ========== storageclass: local ========== + local: + provisioner_image: + tag: 3.4.0 + linux_utils_image: + tag: 3.4.0 + # ========== storageclass: nfs ========== + # nfs provisioner helm version + nfs_provisioner_version: 4.0.10 +dns: + dns_image: + tag: v1.10.1 + dns_cache_image: + tag: 1.22.20 + image_manifests: + - docker.io/calico/apiserver:v3.28.2 + - docker.io/calico/cni:v3.28.2 + - docker.io/calico/csi:v3.28.2 + - docker.io/calico/kube-controllers:v3.28.2 + - docker.io/calico/node-driver-registrar:v3.28.2 + - docker.io/calico/node:v3.28.2 + - docker.io/calico/pod2daemon-flexvol:v3.28.2 + - docker.io/kubesphere/k8s-dns-node-cache:1.22.20 + - docker.io/openebs/provisioner-localpv:3.4.0 + - docker.io/kubesphere/coredns:v1.10.1 + - docker.io/kubesphere/kube-apiserver:{{ .kube_version }} + - docker.io/kubesphere/kube-controller-manager:{{ .kube_version }} + - docker.io/kubesphere/kube-proxy:{{ .kube_version }} + - docker.io/kubesphere/kube-scheduler:{{ .kube_version }} + - docker.io/kubesphere/pause:3.8 + - quay.io/tigera/operator:v1.34.5 + - docker.io/calico/ctl:v3.28.2 + - docker.io/calico/typha:v3.28.2 + - docker.io/calico/apiserver:v3.28.2 + - docker.io/calico/kube-controllers:v3.28.2 + - docker.io/calico/node:v3.28.2 + - docker.io/calico/pod2daemon-flexvol:v3.28.2 + - docker.io/calico/cni:v3.28.2 + - docker.io/calico/node-driver-registrar:v3.28.2 + - docker.io/calico/csi:v3.28.2 diff --git a/builtin/core/roles/defaults/vars/v1.29.yaml b/builtin/core/roles/defaults/vars/v1.29.yaml new file mode 100644 index 000000000..d4dd1b112 --- /dev/null +++ b/builtin/core/roles/defaults/vars/v1.29.yaml @@ -0,0 +1,97 @@ +kubernetes: + # helm binary + helm_version: v3.13.3 +etcd: + # etcd binary + etcd_version: v3.5.10 +image_registry: + # ========== image registry ========== + # keepalived image tag. Used for load balancing when there are multiple image registry nodes. + keepalived_version: 2.0.20 + # ========== image registry: harbor ========== + # harbor image tag + harbor_version: v2.10.1 + # docker-compose binary + dockercompose_version: v2.20.3 + # ========== image registry: docker-registry ========== + # docker-registry image tag + docker_registry_version: 2.8.3 +cri: + # support: containerd,docker + container_manager: containerd + sandbox_image: + tag: "3.8" + # ========== cri ========== + # crictl binary + crictl_version: v1.29.0 + # ========== cri: docker ========== + # docker binary + docker_version: 24.0.7 + # cridockerd. Required when kube_version is greater than 1.24 + cridockerd_version: v0.3.1 + # ========== cri: containerd ========== + # containerd binary + containerd_version: v1.7.6 + # runc binary + runc_version: v1.1.7 +cni: + multus: + image: + tag: v4.1.1 + # ========== cni ========== + # cni_plugins binary (optional) + # cni_plugins_version: v1.2.0 + # ========== cni: calico ========== + # calicoctl binary + calico_version: v3.28.2 + # ========== cni: cilium ========== + # cilium helm + cilium_version: 1.15.4 + # ========== cni: kubeovn ========== + # kubeovn helm + kubeovn_version: 1.13.0 + # ========== cni: hybridnet ========== + # hybridnet helm + hybridnet_version: 0.6.8 +storage_class: + # ========== storageclass ========== + # ========== storageclass: local ========== + local: + provisioner_image: + tag: 3.5.0 + linux_utils_image: + tag: 3.5.0 + # ========== storageclass: nfs ========== + # nfs provisioner helm version + nfs_provisioner_version: 4.1.0 +dns: + dns_image: + tag: v1.11.1 + dns_cache_image: + tag: 1.23.1 + image_manifests: + - docker.io/calico/apiserver:v3.28.2 + - docker.io/calico/cni:v3.28.2 + - docker.io/calico/csi:v3.28.2 + - docker.io/calico/kube-controllers:v3.28.2 + - docker.io/calico/node-driver-registrar:v3.28.2 + - docker.io/calico/node:v3.28.2 + - docker.io/calico/pod2daemon-flexvol:v3.28.2 + - docker.io/kubesphere/k8s-dns-node-cache:1.23.1 + - docker.io/openebs/provisioner-localpv:3.5.0 + - docker.io/kubesphere/coredns:v1.11.1 + - docker.io/kubesphere/kube-apiserver:{{ .kube_version }} + - docker.io/kubesphere/kube-controller-manager:{{ .kube_version }} + - docker.io/kubesphere/kube-proxy:{{ .kube_version }} + - docker.io/kubesphere/kube-scheduler:{{ .kube_version }} + - docker.io/kubesphere/pause:3.8 + - quay.io/tigera/operator:v1.34.5 + - docker.io/calico/ctl:v3.28.2 + - docker.io/calico/typha:v3.28.2 + - docker.io/calico/apiserver:v3.28.2 + - docker.io/calico/kube-controllers:v3.28.2 + - docker.io/calico/node:v3.28.2 + - docker.io/calico/pod2daemon-flexvol:v3.28.2 + - docker.io/calico/cni:v3.28.2 + - docker.io/calico/node-driver-registrar:v3.28.2 + - docker.io/calico/csi:v3.28.2 diff --git a/builtin/core/roles/defaults/vars/v1.30.yaml b/builtin/core/roles/defaults/vars/v1.30.yaml new file mode 100644 index 000000000..21df60fea --- /dev/null +++ b/builtin/core/roles/defaults/vars/v1.30.yaml @@ -0,0 +1,97 @@ +kubernetes: + # helm binary + helm_version: v3.13.3 +etcd: + # etcd binary + etcd_version: v3.5.10 +image_registry: + # ========== image registry ========== + # keepalived image tag. Used for load balancing when there are multiple image registry nodes. + keepalived_version: 2.0.20 + # ========== image registry: harbor ========== + # harbor image tag + harbor_version: v2.10.1 + # docker-compose binary + dockercompose_version: v2.20.3 + # ========== image registry: docker-registry ========== + # docker-registry image tag + docker_registry_version: 2.8.3 +cri: + # support: containerd,docker + container_manager: containerd + sandbox_image: + tag: "3.8" + # ========== cri ========== + # crictl binary + crictl_version: v1.30.0 + # ========== cri: docker ========== + # docker binary + docker_version: 24.0.7 + # cridockerd. Required when kube_version is greater than 1.24 + cridockerd_version: v0.3.1 + # ========== cri: containerd ========== + # containerd binary + containerd_version: v1.7.6 + # runc binary + runc_version: v1.1.7 +cni: + multus: + image: + tag: v4.2.1 + # ========== cni ========== + # cni_plugins binary (optional) + # cni_plugins_version: v1.2.0 + # ========== cni: calico ========== + # calicoctl binary + calico_version: v3.28.2 + # ========== cni: cilium ========== + # cilium helm + cilium_version: 1.15.4 + # ========== cni: kubeovn ========== + # kubeovn helm + kubeovn_version: 1.13.0 + # ========== cni: hybridnet ========== + # hybridnet helm + hybridnet_version: 0.6.8 +storage_class: + # ========== storageclass ========== + # ========== storageclass: local ========== + local: + provisioner_image: + tag: 4.0.0 + linux_utils_image: + tag: 4.0.0 + # ========== storageclass: nfs ========== + # nfs provisioner helm version + nfs_provisioner_version: 4.1.0 +dns: + dns_image: + tag: v1.11.1 + dns_cache_image: + tag: 1.23.1 + image_manifests: + - docker.io/calico/apiserver:v3.28.2 + - docker.io/calico/cni:v3.28.2 + - docker.io/calico/csi:v3.28.2 + - docker.io/calico/kube-controllers:v3.28.2 + - docker.io/calico/node-driver-registrar:v3.28.2 + - docker.io/calico/node:v3.28.2 + - docker.io/calico/pod2daemon-flexvol:v3.28.2 + - docker.io/kubesphere/k8s-dns-node-cache:1.23.1 + - docker.io/openebs/provisioner-localpv:4.0.0 + - docker.io/kubesphere/coredns:v1.11.1 + - docker.io/kubesphere/kube-apiserver:{{ .kube_version }} + - docker.io/kubesphere/kube-controller-manager:{{ .kube_version }} + - docker.io/kubesphere/kube-proxy:{{ .kube_version }} + - docker.io/kubesphere/kube-scheduler:{{ .kube_version }} + - docker.io/kubesphere/pause:3.8 + - quay.io/tigera/operator:v1.34.5 + - docker.io/calico/ctl:v3.28.2 + - docker.io/calico/typha:v3.28.2 + - docker.io/calico/apiserver:v3.28.2 + - docker.io/calico/kube-controllers:v3.28.2 + - docker.io/calico/node:v3.28.2 + - docker.io/calico/pod2daemon-flexvol:v3.28.2 + - docker.io/calico/cni:v3.28.2 + - docker.io/calico/node-driver-registrar:v3.28.2 + - docker.io/calico/csi:v3.28.2 diff --git a/builtin/core/roles/defaults/vars/v1.31.yaml b/builtin/core/roles/defaults/vars/v1.31.yaml new file mode 100644 index 000000000..a1aefd802 --- /dev/null +++ b/builtin/core/roles/defaults/vars/v1.31.yaml @@ -0,0 +1,97 @@ +kubernetes: + # helm binary + helm_version: v3.13.3 +etcd: + # etcd binary + etcd_version: v3.5.11 +image_registry: + # ========== image registry ========== + # keepalived image tag. Used for load balancing when there are multiple image registry nodes. + keepalived_version: 2.0.20 + # ========== image registry: harbor ========== + # harbor image tag + harbor_version: v2.10.1 + # docker-compose binary + dockercompose_version: v2.20.3 + # ========== image registry: docker-registry ========== + # docker-registry image tag + docker_registry_version: 2.8.3 +cri: + # support: containerd,docker + container_manager: containerd + sandbox_image: + tag: "3.8" + # ========== cri ========== + # crictl binary + crictl_version: v1.31.0 + # ========== cri: docker ========== + # docker binary + docker_version: 24.0.7 + # cridockerd. Required when kube_version is greater than 1.24 + cridockerd_version: v0.3.1 + # ========== cri: containerd ========== + # containerd binary + containerd_version: v1.7.6 + # runc binary + runc_version: v1.1.7 +cni: + multus: + image: + tag: v4.2.1 + # ========== cni ========== + # cni_plugins binary (optional) + # cni_plugins_version: v1.2.0 + # ========== cni: calico ========== + # calicoctl binary + calico_version: v3.28.2 + # ========== cni: cilium ========== + # cilium helm + cilium_version: 1.15.4 + # ========== cni: kubeovn ========== + # kubeovn helm + kubeovn_version: 1.13.0 + # ========== cni: hybridnet ========== + # hybridnet helm + hybridnet_version: 0.6.8 +storage_class: + # ========== storageclass ========== + # ========== storageclass: local ========== + local: + provisioner_image: + tag: 4.1.0 + linux_utils_image: + tag: 4.1.0 + # ========== storageclass: nfs ========== + # nfs provisioner helm version + nfs_provisioner_version: 4.2.0 +dns: + dns_image: + tag: v1.11.1 + dns_cache_image: + tag: 1.23.1 + image_manifests: + - docker.io/calico/apiserver:v3.28.2 + - docker.io/calico/cni:v3.28.2 + - docker.io/calico/csi:v3.28.2 + - docker.io/calico/kube-controllers:v3.28.2 + - docker.io/calico/node-driver-registrar:v3.28.2 + - docker.io/calico/node:v3.28.2 + - docker.io/calico/pod2daemon-flexvol:v3.28.2 + - docker.io/kubesphere/k8s-dns-node-cache:1.23.1 + - docker.io/openebs/provisioner-localpv:4.1.0 + - docker.io/kubesphere/coredns:v1.11.1 + - docker.io/kubesphere/kube-apiserver:{{ .kube_version }} + - docker.io/kubesphere/kube-controller-manager:{{ .kube_version }} + - docker.io/kubesphere/kube-proxy:{{ .kube_version }} + - docker.io/kubesphere/kube-scheduler:{{ .kube_version }} + - docker.io/kubesphere/pause:3.8 + - quay.io/tigera/operator:v1.34.5 + - docker.io/calico/ctl:v3.28.2 + - docker.io/calico/typha:v3.28.2 + - docker.io/calico/apiserver:v3.28.2 + - docker.io/calico/kube-controllers:v3.28.2 + - docker.io/calico/node:v3.28.2 + - docker.io/calico/pod2daemon-flexvol:v3.28.2 + - docker.io/calico/cni:v3.28.2 + - docker.io/calico/node-driver-registrar:v3.28.2 + - docker.io/calico/csi:v3.28.2 diff --git a/builtin/core/roles/defaults/vars/v1.32.yaml b/builtin/core/roles/defaults/vars/v1.32.yaml new file mode 100644 index 000000000..4463836d8 --- /dev/null +++ b/builtin/core/roles/defaults/vars/v1.32.yaml @@ -0,0 +1,97 @@ +kubernetes: + # helm binary + helm_version: v3.14.3 +etcd: + # etcd binary + etcd_version: v3.5.11 +image_registry: + # ========== image registry ========== + # keepalived image tag. Used for load balancing when there are multiple image registry nodes. + keepalived_version: 2.0.20 + # ========== image registry: harbor ========== + # harbor image tag + harbor_version: v2.10.1 + # docker-compose binary + dockercompose_version: v2.20.3 + # ========== image registry: docker-registry ========== + # docker-registry image tag + docker_registry_version: 2.8.3 +cri: + # support: containerd,docker + container_manager: containerd + sandbox_image: + tag: "3.8" + # ========== cri ========== + # crictl binary + crictl_version: v1.32.0 + # ========== cri: docker ========== + # docker binary + docker_version: 24.0.7 + # cridockerd. Required when kube_version is greater than 1.24 + cridockerd_version: v0.3.1 + # ========== cri: containerd ========== + # containerd binary + containerd_version: v1.7.6 + # runc binary + runc_version: v1.1.7 +cni: + multus: + image: + tag: v4.3.0 + # ========== cni ========== + # cni_plugins binary (optional) + # cni_plugins_version: v1.2.0 + # ========== cni: calico ========== + # calicoctl binary + calico_version: v3.28.2 + # ========== cni: cilium ========== + # cilium helm + cilium_version: 1.15.4 + # ========== cni: kubeovn ========== + # kubeovn helm + kubeovn_version: 1.13.0 + # ========== cni: hybridnet ========== + # hybridnet helm + hybridnet_version: 0.6.8 +storage_class: + # ========== storageclass ========== + # ========== storageclass: local ========== + local: + provisioner_image: + tag: 4.2.0 + linux_utils_image: + tag: 4.2.0 + # ========== storageclass: nfs ========== + # nfs provisioner helm version + nfs_provisioner_version: 4.2.0 +dns: + dns_image: + tag: v1.11.1 + dns_cache_image: + tag: 1.24.0 + image_manifests: + - docker.io/calico/apiserver:v3.28.2 + - docker.io/calico/cni:v3.28.2 + - docker.io/calico/csi:v3.28.2 + - docker.io/calico/kube-controllers:v3.28.2 + - docker.io/calico/node-driver-registrar:v3.28.2 + - docker.io/calico/node:v3.28.2 + - docker.io/calico/pod2daemon-flexvol:v3.28.2 + - docker.io/kubesphere/k8s-dns-node-cache:1.24.0 + - docker.io/openebs/provisioner-localpv:4.2.0 + - docker.io/kubesphere/coredns:v1.11.1 + - docker.io/kubesphere/kube-apiserver:{{ .kube_version }} + - docker.io/kubesphere/kube-controller-manager:{{ .kube_version }} + - docker.io/kubesphere/kube-proxy:{{ .kube_version }} + - docker.io/kubesphere/kube-scheduler:{{ .kube_version }} + - docker.io/kubesphere/pause:3.8 + - quay.io/tigera/operator:v1.34.5 + - docker.io/calico/ctl:v3.28.2 + - docker.io/calico/typha:v3.28.2 + - docker.io/calico/apiserver:v3.28.2 + - docker.io/calico/kube-controllers:v3.28.2 + - docker.io/calico/node:v3.28.2 + - docker.io/calico/pod2daemon-flexvol:v3.28.2 + - docker.io/calico/cni:v3.28.2 + - docker.io/calico/node-driver-registrar:v3.28.2 + - docker.io/calico/csi:v3.28.2 diff --git a/builtin/core/roles/defaults/vars/v1.33.yaml b/builtin/core/roles/defaults/vars/v1.33.yaml new file mode 100644 index 000000000..94edd6103 --- /dev/null +++ b/builtin/core/roles/defaults/vars/v1.33.yaml @@ -0,0 +1,97 @@ +kubernetes: + # helm binary + helm_version: v3.18.5 +etcd: + # etcd binary + etcd_version: v3.5.11 +image_registry: + # ========== image registry ========== + # keepalived image tag. Used for load balancing when there are multiple image registry nodes. + keepalived_version: 2.0.20 + # ========== image registry: harbor ========== + # harbor image tag + harbor_version: v2.10.1 + # docker-compose binary + dockercompose_version: v2.20.3 + # ========== image registry: docker-registry ========== + # docker-registry image tag + docker_registry_version: 2.8.3 +cri: + # support: containerd,docker + container_manager: containerd + sandbox_image: + tag: "3.9" + # ========== cri ========== + # crictl binary + crictl_version: v1.33.0 + # ========== cri: docker ========== + # docker binary + docker_version: 24.0.7 + # cridockerd. Required when kube_version is greater than 1.24 + cridockerd_version: v0.3.1 + # ========== cri: containerd ========== + # containerd binary + containerd_version: v1.7.6 + # runc binary + runc_version: v1.1.7 +cni: + multus: + image: + tag: v4.3.0 + # ========== cni ========== + # cni_plugins binary (optional) + # cni_plugins_version: v1.2.0 + # ========== cni: calico ========== + # calicoctl binary + calico_version: v3.28.2 + # ========== cni: cilium ========== + # cilium helm + cilium_version: 1.15.4 + # ========== cni: kubeovn ========== + # kubeovn helm + kubeovn_version: 1.13.0 + # ========== cni: hybridnet ========== + # hybridnet helm + hybridnet_version: 0.6.8 +storage_class: + # ========== storageclass ========== + # ========== storageclass: local ========== + local: + provisioner_image: + tag: 4.2.0 + linux_utils_image: + tag: 4.2.0 + # ========== storageclass: nfs ========== + # nfs provisioner helm version + nfs_provisioner_version: 4.3.0 +dns: + dns_image: + tag: v1.12.1 + dns_cache_image: + tag: 1.24.0 + image_manifests: + - docker.io/calico/apiserver:v3.28.2 + - docker.io/calico/cni:v3.28.2 + - docker.io/calico/csi:v3.28.2 + - docker.io/calico/kube-controllers:v3.28.2 + - docker.io/calico/node-driver-registrar:v3.28.2 + - docker.io/calico/node:v3.28.2 + - docker.io/calico/pod2daemon-flexvol:v3.28.2 + - docker.io/kubesphere/k8s-dns-node-cache:1.24.0 + - docker.io/openebs/provisioner-localpv:4.2.0 + - docker.io/kubesphere/coredns:v1.12.1 + - docker.io/kubesphere/kube-apiserver:{{ .kube_version }} + - docker.io/kubesphere/kube-controller-manager:{{ .kube_version }} + - docker.io/kubesphere/kube-proxy:{{ .kube_version }} + - docker.io/kubesphere/kube-scheduler:{{ .kube_version }} + - docker.io/kubesphere/pause:3.9 + - quay.io/tigera/operator:v1.34.5 + - docker.io/calico/ctl:v3.28.2 + - docker.io/calico/typha:v3.28.2 + - docker.io/calico/apiserver:v3.28.2 + - docker.io/calico/kube-controllers:v3.28.2 + - docker.io/calico/node:v3.28.2 + - docker.io/calico/pod2daemon-flexvol:v3.28.2 + - docker.io/calico/cni:v3.28.2 + - docker.io/calico/node-driver-registrar:v3.28.2 + - docker.io/calico/csi:v3.28.2 diff --git a/builtin/core/roles/download/tasks/binary.yaml b/builtin/core/roles/download/tasks/binary.yaml new file mode 100644 index 000000000..768732ada --- /dev/null +++ b/builtin/core/roles/download/tasks/binary.yaml @@ -0,0 +1,288 @@ +--- +- name: Binary | Ensure etcd binary is present + tags: ["etcd"] + loop: "{{ .download.arch | toJson }}" + when: + - .etcd.deployment_type | eq "external" + - .etcd.etcd_version | empty | not + command: | + artifact_name={{ get .download.artifact_url.etcd .item | splitList "/" | last }} + artifact_path={{ .binary_dir }}/etcd/{{ .etcd.etcd_version }}/{{ .item }} + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # Attempt to download etcd binary + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .download.artifact_url.etcd .item }}) + if [ $http_code != 200 ]; then + echo "Failed to download etcd binary. HTTP status code: $http_code" + exit 1 + fi + curl -L -o $artifact_path/$artifact_name {{ get .download.artifact_url.etcd .item }} + fi + +- name: Binary | Ensure Kubernetes binaries are present + tags: ["kubernetes"] + loop: "{{ .download.arch | toJson }}" + when: .kubernetes.kube_version | empty | not + command: | + kube_path={{ .binary_dir }}/kube/{{ .kubernetes.kube_version }}/{{ .item }} + if [ ! -f $kube_path/kubelet ]; then + mkdir -p $kube_path + # Download kubelet if missing + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .download.artifact_url.kubelet .item }}) + if [ $http_code != 200 ]; then + echo "Failed to download kubelet. HTTP status code: $http_code" + exit 1 + fi + curl -L -o $kube_path/kubelet {{ get .download.artifact_url.kubelet .item }} + fi + if [ ! -f $kube_path/kubeadm ]; then + mkdir -p $kube_path + # Download kubeadm if missing + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .download.artifact_url.kubeadm .item }}) + if [ $http_code != 200 ]; then + echo "Failed to download kubeadm. HTTP status code: $http_code" + exit 1 + fi + curl -L -o $kube_path/kubeadm {{ get .download.artifact_url.kubeadm .item }} + fi + if [ ! -f $kube_path/kubectl ]; then + mkdir -p $kube_path + # Download kubectl if missing + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .download.artifact_url.kubectl .item }}) + if [ $http_code != 200 ]; then + echo "Failed to download kubectl. HTTP status code: $http_code" + exit 1 + fi + curl -L -o $kube_path/kubectl {{ get .download.artifact_url.kubectl .item }} + fi + +- name: Binary | Ensure CNI plugins are present + tags: ["kubernetes"] + loop: "{{ .download.arch | toJson }}" + when: .cni.cni_plugins_version | empty | not + command: | + artifact_name={{ get .download.artifact_url.cni_plugins .item | splitList "/" | last }} + artifact_path={{ .binary_dir }}/cni/plugins/{{ .cni.cni_plugins_version }}/{{ .item }} + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # Attempt to download CNI plugins + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .download.artifact_url.cni_plugins .item }}) + if [ $http_code != 200 ]; then + echo "Failed to download CNI plugins. HTTP status code: $http_code" + exit 1 + fi + curl -L -o $artifact_path/$artifact_name {{ get .download.artifact_url.cni_plugins .item }} + fi + +- name: Binary | Ensure Helm binary is present + tags: ["kubernetes"] + loop: "{{ .download.arch | toJson }}" + when: .kubernetes.helm_version | empty | not + command: | + artifact_name={{ get .download.artifact_url.helm .item | splitList "/" | last }} + artifact_path={{ .binary_dir }}/helm/{{ .kubernetes.helm_version }}/{{ .item }} + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # Attempt to download Helm binary + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .download.artifact_url.helm .item }}) + if [ $http_code != 200 ]; then + echo "Failed to download Helm binary. HTTP status code: $http_code" + exit 1 + fi + curl -L -o $artifact_path/$artifact_name {{ get .download.artifact_url.helm .item }} + fi + +- name: Binary | Ensure crictl binary is present + tags: ["kubernetes"] + loop: "{{ .download.arch | toJson }}" + when: .cri.crictl_version | empty | not + command: | + artifact_name={{ get .download.artifact_url.crictl .item | splitList "/" | last }} + artifact_path={{ .binary_dir }}/crictl/{{ .cri.crictl_version }}/{{ .item }} + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # Attempt to download crictl binary + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .download.artifact_url.crictl .item }}) + if [ $http_code != 200 ]; then + echo "Failed to download crictl binary. HTTP status code: $http_code" + exit 1 + fi + curl -L -o $artifact_path/$artifact_name {{ get .download.artifact_url.crictl .item }} + fi + +- name: Binary | Ensure Docker binary is present + tags: ["kubernetes","image_registry"] + loop: "{{ .download.arch | toJson }}" + when: + - .cri.docker_version | empty | not + - or (.image_registry.type | empty | not) (.cri.container_manager | eq "docker") + command: | + artifact_name={{ get .download.artifact_url.docker .item | splitList "/" | last }} + artifact_path={{ .binary_dir }}/docker/{{ .cri.docker_version }}/{{ .item }} + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # Attempt to download Docker binary + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .download.artifact_url.docker .item }}) + if [ $http_code != 200 ]; then + echo "Failed to download Docker binary. HTTP status code: $http_code" + exit 1 + fi + curl -L -o $artifact_path/$artifact_name {{ get .download.artifact_url.docker .item }} + fi + +- name: Binary | Ensure cri-dockerd binary is present + tags: ["kubernetes"] + loop: "{{ .download.arch | toJson }}" + when: + - .cri.cridockerd_version | empty | not + - .cri.container_manager | eq "docker" + - .kubernetes.kube_version | semverCompare ">=v1.24.0" + command: | + artifact_name={{ get .download.artifact_url.cridockerd .item | splitList "/" | last }} + artifact_path={{ .binary_dir }}/cri-dockerd/{{ .cri.cridockerd_version }}/{{ .item }} + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # Attempt to download cri-dockerd binary + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .download.artifact_url.cridockerd .item }}) + if [ $http_code != 200 ]; then + echo "Failed to download cri-dockerd binary. HTTP status code: $http_code" + exit 1 + fi + curl -L -o $artifact_path/$artifact_name {{ get .download.artifact_url.cridockerd .item }} + fi + +- name: Binary | Ensure containerd binary is present + tags: ["kubernetes"] + loop: "{{ .download.arch | toJson }}" + when: + - .cri.containerd_version | empty | not + - .cri.container_manager | eq "containerd" + command: | + artifact_name={{ get .download.artifact_url.containerd .item | splitList "/" | last }} + artifact_path={{ .binary_dir }}/containerd/{{ .cri.containerd_version }}/{{ .item }} + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # Attempt to download containerd binary + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .download.artifact_url.containerd .item }}) + if [ $http_code != 200 ]; then + echo "Failed to download containerd binary. HTTP status code: $http_code" + exit 1 + fi + curl -L -o $artifact_path/$artifact_name {{ get .download.artifact_url.containerd .item }} + fi + +- name: Binary | Ensure runc binary is present + tags: ["kubernetes"] + loop: "{{ .download.arch | toJson }}" + when: + - .cri.runc_version | empty | not + - .cri.container_manager | eq "containerd" + command: | + artifact_name={{ get .download.artifact_url.runc .item | splitList "/" | last }} + artifact_path={{ .binary_dir }}/runc/{{ .cri.runc_version }}/{{ .item }} + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # Attempt to download runc binary + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .download.artifact_url.runc .item }}) + if [ $http_code != 200 ]; then + echo "Failed to download runc binary. HTTP status code: $http_code" + exit 1 + fi + curl -L -o $artifact_path/$artifact_name {{ get .download.artifact_url.runc .item }} + fi + +- name: Binary | Ensure calicoctl binary is present + tags: ["kubernetes"] + loop: "{{ .download.arch | toJson }}" + when: + - .cni.calico_version | empty | not + - .cni.type | eq "calico" + command: | + artifact_name=calicoctl + artifact_path={{ .binary_dir }}/cni/calico/{{ .cni.calico_version }}/{{ .item }} + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # Attempt to download calicoctl binary + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .download.artifact_url.calicoctl .item }}) + if [ $http_code != 200 ]; then + echo "Failed to download calicoctl binary. HTTP status code: $http_code" + exit 1 + fi + curl -L -o $artifact_path/$artifact_name {{ get .download.artifact_url.calicoctl .item }} + fi + +- name: Binary | Ensure Docker Registry binary is present + tags: ["image_registry"] + loop: "{{ .download.arch | toJson }}" + when: + - .image_registry.docker_registry_version | empty | not + - .image_registry.type | eq "docker-registry" + command: | + artifact_name={{ get .download.artifact_url.docker_registry .item | splitList "/" | last }} + artifact_path={{ .binary_dir }}/image-registry/docker-registry/{{ .image_registry.docker_registry_version }}/{{ .item }} + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # Attempt to download Docker Registry binary + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .download.artifact_url.docker_registry .item }}) + if [ $http_code != 200 ]; then + echo "Failed to download Docker Registry binary. HTTP status code: $http_code" + exit 1 + fi + curl -L -o $artifact_path/$artifact_name {{ get .download.artifact_url.docker_registry .item }} + fi + +- name: Binary | Ensure docker-compose binary is present + tags: ["image_registry"] + loop: "{{ .download.arch | toJson }}" + when: + - .cri.dockercompose_version | empty | not + - .image_registry.type | eq "harbor" + command: | + compose_name=docker-compose + compose_path={{ .binary_dir }}/image-registry/docker-compose/{{ .cri.dockercompose_version }}/{{ .item }} + if [ ! -f $compose_path/$compose_name ]; then + mkdir -p $compose_path + # Attempt to download docker-compose binary + curl -L -o $compose_path/$compose_name {{ get .download.artifact_url.dockercompose .item }} + fi + +- name: Binary | Ensure Harbor binary is present + tags: ["image_registry"] + loop: "{{ .download.arch | toJson }}" + when: + - .image_registry.harbor_version | empty | not + - .image_registry.type | eq "harbor" + command: | + harbor_name={{ get .download.artifact_url.harbor .item | splitList "/" | last }} + harbor_path={{ .binary_dir }}/image-registry/harbor/{{ .image_registry.harbor_version }}/{{ .item }} + if [ ! -f $harbor_path/$harbor_name ]; then + mkdir -p $harbor_path + # Attempt to download Harbor binary + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .download.artifact_url.harbor .item }}) + if [ $http_code != 200 ]; then + echo "Failed to download Harbor binary. HTTP status code: $http_code" + exit 1 + fi + curl -L -o $harbor_path/$harbor_name {{ get .download.artifact_url.harbor .item }} + fi + +- name: Binary | Ensure keepalived binary is present + tags: ["image_registry"] + loop: "{{ .download.arch | toJson }}" + when: + - .image_registry.keepalived_version | empty | not + - .image_registry.ha_vip | empty | not + - .groups.image_registry | len | lt 1 + command: | + artifact_name={{ get .download.artifact_url.keepalived .item | splitList "/" | last }} + artifact_path={{ .binary_dir }}/image-registry/keepalived/{{ .keepalived_version }}/{{ .item }} + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # Attempt to download keepalived binary + http_code=$(curl -Lo /dev/null -s -w "%{http_code}" {{ get .download.artifact_url.keepalived .item }}) + if [ $http_code != 200 ]; then + echo "Failed to download keepalived binary. HTTP status code: $http_code" + exit 1 + fi + curl -L -o $artifact_path/$artifact_name {{ get .download.artifact_url.keepalived .item }} + fi diff --git a/builtin/core/roles/download/tasks/helm.yaml b/builtin/core/roles/download/tasks/helm.yaml new file mode 100644 index 000000000..29155e271 --- /dev/null +++ b/builtin/core/roles/download/tasks/helm.yaml @@ -0,0 +1,78 @@ +--- +- name: Helm | Ensure the Calico binary is available + when: + - .cni.calico_version | empty | not + - .cni.type | eq "calico" + command: | + artifact_name={{ .download.artifact_url.calico | splitList "/" | last }} + artifact_path={{ .binary_dir }}/cni/calico + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # Download the Calico binary if it does not exist + curl -Lo $artifact_path/$artifact_name {{ .download.artifact_url.calico }} + fi + +- name: Helm | Ensure the Cilium binary is available + when: + - .cni.cilium_version | empty | not + - .cni.type | eq "cilium" + command: | + artifact_name={{ .download.artifact_url.cilium | splitList "/" | last }} + artifact_path={{ .binary_dir }}/cni/cilium + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # Download the Cilium binary if it does not exist + curl -Lo $artifact_path/$artifact_name {{ .download.artifact_url.cilium }} + fi + +- name: Helm | Ensure the Flannel binary is available + when: + - .cni.flannel_version | empty | not + - .cni.type | eq "flannel" + command: | + artifact_name={{ .download.artifact_url.flannel | splitList "/" | last }} + artifact_path={{ .binary_dir }}/cni/flannel + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # Download the Flannel binary if it does not exist + curl -Lo $artifact_path/$artifact_name {{ .download.artifact_url.flannel }} + fi + +- name: Helm | Ensure the Kube-OVN binary is available + when: + - .kubeovn_version | empty | not + - .cni.type | eq "kubeovn" + command: | + artifact_name={{ .download.artifact_url.kubeovn | splitList "/" | last }} + artifact_path={{ .binary_dir }}/cni/kubeovn + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # Download the Kube-OVN binary if it does not exist + curl -Lo $artifact_path/$artifact_name {{ .download.artifact_url.kubeovn }} + fi + +- name: Helm | Ensure the Hybridnet binary is available + when: + - .cni.hybridnet_version | empty | not + - .cni.type | eq "hybridnet" + command: | + artifact_name={{ .download.artifact_url.hybridnet | splitList "/" | last }} + artifact_path={{ .binary_dir }}/cni/hybridnet + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # Download the Hybridnet binary if it does not exist + curl -Lo $artifact_path/$artifact_name {{ .download.artifact_url.hybridnet }} + fi + +- name: Helm | Ensure the NFS Provisioner binary is available + when: + - .storage_class.nfs_provisioner_version | empty | not + - .storage_class.nfs.enabled + command: | + artifact_name={{ .download.artifact_url.nfs_provisioner | splitList "/" | last }} + artifact_path={{ .binary_dir }}/sc + if [ ! -f $artifact_path/$artifact_name ]; then + mkdir -p $artifact_path + # Download the NFS Provisioner binary if it does not exist + curl -Lo $artifact_path/$artifact_name {{ .download.artifact_url.nfs_provisioner }} + fi diff --git a/builtin/core/roles/download/tasks/images.yaml b/builtin/core/roles/download/tasks/images.yaml new file mode 100644 index 000000000..6d7183abc --- /dev/null +++ b/builtin/core/roles/download/tasks/images.yaml @@ -0,0 +1,11 @@ +- name: Image | Download container images + image: + pull: + username: "{{ .image_registry.auth.username }}" + password: "{{ .image_registry.auth.password }}" + images_dir: >- + {{ .binary_dir }}/images/ + manifests: "{{ .image_manifests | toJson }}" + when: + - .image_manifests | default list | empty | not + - .download.download_image \ No newline at end of file diff --git a/builtin/core/roles/download/tasks/main.yaml b/builtin/core/roles/download/tasks/main.yaml new file mode 100644 index 000000000..d0bce27d2 --- /dev/null +++ b/builtin/core/roles/download/tasks/main.yaml @@ -0,0 +1,27 @@ +--- +- name: Artifact | Extract artifact archive to working directory + tags: ["always"] + command: | + if [ -f "{{ .download.artifact_file }}" ]; then + mkdir -p {{ .binary_dir }} + tar -zxvf {{ .artifact_file }} -C {{ .binary_dir }} + fi + when: .download.artifact_file | empty | not + +- name: Artifact | Download required binaries and images + when: .download.artifact_file | empty + block: + # Download core binaries + - include_tasks: binary.yaml + # Download Helm and CNI binaries + - include_tasks: helm.yaml + tags: ["kubernetes"] + # Download remote images to the local images directory + - include_tasks: images.yaml + tags: ["kubernetes", "image_registry"] + +- name: Artifact | Set ownership of working directory to sudo user + tags: ["always"] + ignore_errors: true + command: | + chown -R ${SUDO_UID}:${SUDO_GID} {{ .work_dir }} diff --git a/builtin/core/roles/etcd/files/backup.service b/builtin/core/roles/etcd/files/backup.service new file mode 100644 index 000000000..a03f4226a --- /dev/null +++ b/builtin/core/roles/etcd/files/backup.service @@ -0,0 +1,5 @@ +[Unit] +Description=Backup ETCD +[Service] +Type=oneshot +ExecStart=/usr/local/bin/kube-scripts/backup_etcd.sh diff --git a/builtin/core/roles/etcd/files/etcd.service b/builtin/core/roles/etcd/files/etcd.service new file mode 100644 index 000000000..178c3e246 --- /dev/null +++ b/builtin/core/roles/etcd/files/etcd.service @@ -0,0 +1,18 @@ +[Unit] +Description=etcd +After=network.target + +[Service] +User=root +Type=notify +Nice=-20 +OOMScoreAdjust=-1000 +EnvironmentFile=/etc/etcd.env +ExecStart=/usr/local/bin/etcd +NotifyAccess=all +RestartSec=10s +LimitNOFILE=40000 +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/builtin/core/roles/etcd/tasks/backup_service.yaml b/builtin/core/roles/etcd/tasks/backup_service.yaml new file mode 100644 index 000000000..6975bdd02 --- /dev/null +++ b/builtin/core/roles/etcd/tasks/backup_service.yaml @@ -0,0 +1,21 @@ +--- +- name: Backup | Synchronize custom etcd backup script + template: + src: >- + {{ .etcd.backup.etcd_backup_script }} + dest: /usr/local/bin/kube-scripts/backup_etcd.sh + mode: 777 + +- name: Backup | Deploy systemd service for etcd backup + copy: + src: backup.service + dest: /etc/systemd/system/backup-etcd.service + +- name: Backup | Deploy systemd timer for scheduled etcd backup + template: + src: backup.timer + dest: /etc/systemd/system/backup-etcd.timer + +- name: Backup | Reload systemd and enable etcd backup timer + command: | + systemctl daemon-reload && systemctl enable --now backup-etcd.timer diff --git a/builtin/core/roles/etcd/tasks/expansion.yaml b/builtin/core/roles/etcd/tasks/expansion.yaml new file mode 100644 index 000000000..e2fbea33f --- /dev/null +++ b/builtin/core/roles/etcd/tasks/expansion.yaml @@ -0,0 +1,37 @@ +- name: Expansion | Expand cluster on existing etcd nodes + when: .etcd_install_LoadState.stdout | eq "loaded" + block: + - name: Expansion | Update /etc/etcd.env configuration file + template: + src: etcd.env + dest: /etc/etcd.env + - name: Expansion | Restart etcd service + command: | + systemctl restart etcd.service + - name: Expansion | Verify etcd service becomes healthy within 1 minute + command: | + for ((i=1; i<=12; i++)); do + if ETCDCTL_API=3 etcdctl \ + --endpoints=https://localhost:2379 \ + --cacert=/etc/ssl/etcd/ssl/ca.crt \ + --cert=/etc/ssl/etcd/ssl/server.crt \ + --key=/etc/ssl/etcd/ssl/server.key \ + endpoint health >/dev/null 2>&1; then + echo "✅ etcd is health" + exit 0 + fi + sleep 5 + done + echo "❌ etcd etcd is not health in 1 minute" + exit 1 + +- name: Expansion | Add new etcd member from non-installed node + when: .etcd_install_LoadState.stdout | eq "not-found" + delegate_to: "{{ .installed_etcd }}" + command: | + ETCDCTL_API=3 etcdctl \ + --endpoints=https://localhost:2379 \ + --cacert=/etc/ssl/etcd/ssl/ca.crt \ + --cert=/etc/ssl/etcd/ssl/server.crt \ + --key=/etc/ssl/etcd/ssl/server.key \ + member add {{ .inventory_hostname }} \ No newline at end of file diff --git a/builtin/core/roles/etcd/tasks/install.yaml b/builtin/core/roles/etcd/tasks/install.yaml new file mode 100644 index 000000000..f562ceb52 --- /dev/null +++ b/builtin/core/roles/etcd/tasks/install.yaml @@ -0,0 +1,41 @@ +--- +- name: Install | Initialize etcd environment + block: + - name: Install | Create etcd system user + command: | + id etcd &>/dev/null || useradd -M -c 'Etcd user' -s /sbin/nologin -r etcd || : + - name: Install | Create etcd data directory and set ownership + command: | + if [ ! -d "{{ .item }}" ]; then + mkdir -p {{ .item }} && chown -R etcd {{ .item }} + fi + loop: + - "{{ .etcd.env.data_dir }}" + +- name: Install | Generate etcd environment configuration file + template: + src: etcd.env + dest: /etc/etcd.env + +- name: Install | Deploy etcd systemd service file + copy: + src: etcd.service + dest: /etc/systemd/system/etcd.service + +# refer: https://etcd.io/docs/v3.5/tuning/ +- name: Install | Set CPU governor to performance mode + command: | + echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor + when: .etcd.performance + +- name: Install | Configure network traffic priority for etcd + command: | + tc qdisc add dev eth0 root handle 1: prio bands 3 + tc filter add dev eth0 parent 1: protocol ip prio 1 u32 match ip sport 2380 0xffff flowid 1:1 + tc filter add dev eth0 parent 1: protocol ip prio 1 u32 match ip dport 2380 0xffff flowid 1:1 + tc filter add dev eth0 parent 1: protocol ip prio 2 u32 match ip sport 2379 0xffff flowid 1:1 + tc filter add dev eth0 parent 1: protocol ip prio 2 u32 match ip dport 2379 0xffff flowid 1:1 + when: .etcd.traffic_priority + +- name: Install | Start and enable etcd systemd service + command: systemctl daemon-reload && systemctl start etcd && systemctl enable etcd diff --git a/builtin/core/roles/etcd/tasks/main.yaml b/builtin/core/roles/etcd/tasks/main.yaml new file mode 100644 index 000000000..167fccc78 --- /dev/null +++ b/builtin/core/roles/etcd/tasks/main.yaml @@ -0,0 +1,20 @@ +--- +- include_tasks: prepare.yaml + +- name: ETCD | Upgrade etcd if a newer version is available + when: + - .etcd_install_LoadState.stdout | eq "loaded" + - .etcd.etcd_version | semverCompare (printf ">v%s" (index .etcd_install_version "stdout" "etcd Version")) + include_tasks: upgrade.yaml + +- name: ETCD | Expand the etcd cluster by adding new nodes if required + when: + - .installed_etcd | empty | not + - .need_installed_etcd | fromJson | empty | not + include_tasks: expansion.yaml + +- name: ETCD | Install etcd and set up the backup service if not already present + when: .etcd_install_LoadState.stdout | eq "not-found" + block: + - include_tasks: install.yaml + - include_tasks: backup_service.yaml diff --git a/builtin/core/roles/etcd/tasks/prepare.yaml b/builtin/core/roles/etcd/tasks/prepare.yaml new file mode 100644 index 000000000..628a32e41 --- /dev/null +++ b/builtin/core/roles/etcd/tasks/prepare.yaml @@ -0,0 +1,81 @@ +--- +- name: Prepare | Ensure installed etcd is running and healthy + when: .etcd_install_LoadState.stdout | eq "loaded" + assert: + that: .etcd_install_ActiveState.stdout | eq "active" + fail_msg: >- + etcd service is installed but not running + +- name: Prepare | Set etcd node parameters + block: + - name: Prepare | Identify nodes with installed or missing etcd + run_once: true + add_hostvars: + hosts: etcd + vars: + installed_etcd: >- + {{- $needInstalled := list -}} + {{- range .groups.etcd -}} + {{- if (index $.hostvars . "etcd_install_LoadState" "stdout") | eq "loaded" -}} + {{- $needInstalled = append $needInstalled . -}} + {{- end -}} + {{- end -}} + {{ $needInstalled | first | default "" }} + need_installed_etcd: >- + {{- $needInstalled := list -}} + {{- range .groups.etcd -}} + {{- if (index $.hostvars . "etcd_install_LoadState" "stdout") | eq "not-found" -}} + {{- $needInstalled = append $needInstalled . -}} + {{- end -}} + {{- end -}} + {{ $needInstalled | toJson }} + +- name: Prepare | Check installed etcd version + when: .etcd_install_LoadState.stdout | eq "loaded" + block: + - name: Prepare | Ensure target etcd version is not lower than installed version + when: .etcd_install_LoadState.stdout | eq "loaded" + assert: + that: .etcd.etcd_version | semverCompare (printf ">=v%s" (index .etcd_install_version "stdout" "etcd Version")) + fail_msg: >- + Installed etcd version: {{ index .etcd_install_version "stdout" "etcd Version" }} is lower than target etcd version: {{ .etcd.etcd_version }} + +- name: Prepare | Synchronize etcd package to node if new install or upgrade + when: + - or (.etcd_install_version.error | empty | not) (.etcd.etcd_version | semverCompare (printf ">v%s" (index .etcd_install_version "stdout" "etcd Version"))) + block: + - name: Prepare | Copy etcd binary package to remote node + copy: + src: >- + {{ .binary_dir }}/etcd/{{ .etcd.etcd_version }}/{{ .binary_type }}/etcd-{{ .etcd.etcd_version }}-linux-{{ .binary_type }}.tar.gz + dest: >- + {{ .tmp_dir }}/etcd-{{ .etcd.etcd_version }}-linux-{{ .binary_type }}.tar.gz + - name: Prepare | Extract etcd binary package to /usr/local/bin/ + command: | + tar --strip-components=1 -C /usr/local/bin/ -xvf {{ .tmp_dir }}/etcd-{{ .etcd.etcd_version }}-linux-{{ .binary_type }}.tar.gz \ + --wildcards 'etcd-{{ .etcd.etcd_version }}-linux-{{ .binary_type }}/etcd*' + +- name: Prepare | Synchronize certificates to node for new install or expansion + when: >- + or + (.etcd_install_version.error | empty | not) + (and + (.installed_etcd | empty | not) + (.need_installed_etcd | fromJson | empty | not) + ) + block: + - name: Prepare | Copy CA certificate to etcd node + copy: + src: >- + {{ .etcd.ca_file }} + dest: /etc/ssl/etcd/ssl/ca.crt + - name: Prepare | Copy server certificate to etcd node + copy: + src: >- + {{ .etcd.cert_file }} + dest: /etc/ssl/etcd/ssl/server.crt + - name: Prepare | Copy server key to etcd node + copy: + src: >- + {{ .etcd.key_file }} + dest: /etc/ssl/etcd/ssl/server.key diff --git a/builtin/core/roles/etcd/tasks/upgrade.yaml b/builtin/core/roles/etcd/tasks/upgrade.yaml new file mode 100644 index 000000000..e969b3bdd --- /dev/null +++ b/builtin/core/roles/etcd/tasks/upgrade.yaml @@ -0,0 +1,23 @@ +- name: Upgrade | Backup etcd data before upgrade + command: BACKUP_DIR="{{ .etcd.backup.backup_dir }}/install/etcd-v{{ index .etcd_install_version "stdout" "etcd Version" }}-$(date +%Y-%m-%d-%H-%M-%S)" /usr/local/bin/kube-scripts/backup_etcd.sh + +- name: Upgrade | Restart etcd service after upgrade + command: | + systemctl restart etcd.service + +- name: Upgrade | Ensure etcd service becomes healthy within 1 minute + command: | + for ((i=1; i<=12; i++)); do + if ETCDCTL_API=3 etcdctl \ + --endpoints=https://localhost:2379 \ + --cacert=/etc/ssl/etcd/ssl/ca.crt \ + --cert=/etc/ssl/etcd/ssl/server.crt \ + --key=/etc/ssl/etcd/ssl/server.key \ + endpoint health >/dev/null 2>&1; then + echo "✅ etcd is healthy" + exit 0 + fi + sleep 5 + done + echo "❌ etcd is not healthy within 1 minute" + exit 1 diff --git a/builtin/core/roles/etcd/templates/backup.sh b/builtin/core/roles/etcd/templates/backup.sh new file mode 100644 index 000000000..b2e3df85b --- /dev/null +++ b/builtin/core/roles/etcd/templates/backup.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +ETCDCTL_PATH='/usr/local/bin/etcdctl' +{{- if .internal_ipv4 | empty | not }} +ENDPOINTS='https://{{ .internal_ipv4 }}:2379' +{{- else if .internal_ipv6 | empty | not }} +ENDPOINTS='https://{{ .internal_ipv6 }}:2379' +{{- end }} +ETCD_DATA_DIR="{{ .etcd.env.data_dir }}" +BACKUP_DIR="${BACKUP_DIR:-{{ .etcd.backup.backup_dir }}/timer/etcd-$(date +%Y-%m-%d-%H-%M-%S)}" +KEEPBACKUPNUMBER='{{ .etcd.backup.keep_backup_number }}' +((KEEPBACKUPNUMBER++)) + +ETCDCTL_CERT="/etc/ssl/etcd/ssl/server.crt" +ETCDCTL_KEY="/etc/ssl/etcd/ssl/server.key" +ETCDCTL_CA_FILE="/etc/ssl/etcd/ssl/ca.crt" + +[ ! -d $BACKUP_DIR ] && mkdir -p $BACKUP_DIR + +# export ETCDCTL_API=3;$ETCDCTL_PATH backup --data-dir $ETCD_DATA_DIR --backup-dir $BACKUP_DIR + +# sleep 3 + +{ +export ETCDCTL_API=3;$ETCDCTL_PATH --endpoints="$ENDPOINTS" snapshot save $BACKUP_DIR/snapshot.db \ + --cacert="$ETCDCTL_CA_FILE" \ + --cert="$ETCDCTL_CERT" \ + --key="$ETCDCTL_KEY" +} > /dev/null + +sleep 3 + +cd $BACKUP_DIR/../ && ls -lt |awk '{if(NR > '$KEEPBACKUPNUMBER'){print "rm -rf "$9}}'|sh diff --git a/builtin/core/roles/etcd/templates/backup.timer b/builtin/core/roles/etcd/templates/backup.timer new file mode 100644 index 000000000..4b73c6bc2 --- /dev/null +++ b/builtin/core/roles/etcd/templates/backup.timer @@ -0,0 +1,7 @@ +[Unit] +Description=Timer to backup ETCD +[Timer] +OnCalendar={{ .etcd.backup.on_calendar }} +Unit=backup-etcd.service +[Install] +WantedBy=multi-user.target diff --git a/builtin/core/roles/etcd/templates/etcd.env b/builtin/core/roles/etcd/templates/etcd.env new file mode 100644 index 000000000..a9a74707d --- /dev/null +++ b/builtin/core/roles/etcd/templates/etcd.env @@ -0,0 +1,75 @@ +{{- $ips := list -}} +{{- $state := "new" -}} +{{- range .groups.etcd | default list -}} + {{- $internalIPv4 := index $.hostvars . "internal_ipv4" | default "" -}} + {{- $internalIPv6 := index $.hostvars . "internal_ipv6" | default "" -}} + {{- if $internalIPv4 | empty | not -}} + {{- $ips = append $ips (printf "%s=https://%s:2380" (index $.hostvars . "hostname") $internalIPv4) -}} + {{- else if $internalIPv6 | empty | not }} + {{- $ips = append $ips (printf "%s=https://%s:2380" (index $.hostvars . "hostname") $internalIPv6) -}} + {{- end -}} + {{ if index $.hostvars . "etcd_install_LoadState" "stdout" | eq "loaded" -}} + {{- $state := "existing" -}} + {{- end -}} +{{- end -}} +ETCD_DATA_DIR={{ .etcd.env.data_dir }} +ETCD_INITIAL_CLUSTER_STATE={{ $state }} +ETCD_INITIAL_CLUSTER_TOKEN={{ .etcd.env.token }} +{{- if .internal_ipv4 | empty | not }} +ETCD_ADVERTISE_CLIENT_URLS={{ printf "https://%s:2379" .internal_ipv4 }} +ETCD_INITIAL_ADVERTISE_PEER_URLS={{ printf "https://%s:2380" .internal_ipv4 }} +ETCD_LISTEN_CLIENT_URLS={{ printf "https://%s:2379" .internal_ipv4 }},https://127.0.0.1:2379 +ETCD_LISTEN_PEER_URLS={{ printf "https://%s:2380" .internal_ipv4 }} +{{- else if .internal_ipv6 | empty | not }} +ETCD_ADVERTISE_CLIENT_URLS={{ printf "https://%s:2379" .internal_ipv6 }} +ETCD_INITIAL_ADVERTISE_PEER_URLS={{ printf "https://%s:2380" .internal_ipv6 }} +ETCD_LISTEN_CLIENT_URLS={{ printf "https://%s:2379" .internal_ipv6 }},https://::1:2379 +ETCD_LISTEN_PEER_URLS={{ printf "https://%s:2380" .internal_ipv6 }} +{{- end }} + +ETCD_NAME={{ .hostname }} +ETCD_PROXY=off +ETCD_ENABLE_V2=true +ETCD_INITIAL_CLUSTER={{ $ips | join "," }} +ETCD_ELECTION_TIMEOUT={{ .etcd.env.election_timeout }} +ETCD_HEARTBEAT_INTERVAL={{ .etcd.env.heartbeat_interval }} +ETCD_AUTO_COMPACTION_RETENTION={{ .etcd.env.compaction_retention }} +ETCD_SNAPSHOT_COUNT={{ .etcd.env.snapshot_count }} +{{- if .etcd.metrics }} +ETCD_METRICS={{ .etcd.env.metrics }} +{{- end }} +{{- if .etcd.env.quota_backend_bytes }} +ETCD_QUOTA_BACKEND_BYTES={{ .etcd.env.quota_backend_bytes }} +{{- end }} +{{- if .etcd.env.max_request_bytes }} +ETCD_MAX_REQUEST_BYTES={{ .etcd.env.max_request_bytes }} +{{- end }} +{{- if .etcd.env.max_snapshots }} +ETCD_MAX_SNAPSHOTS={{ .etcd.env.max_snapshots }} +{{- end }} +{{- if .etcd.env.max_wals }} +ETCD_MAX_WALS={{ .etcd.env.max_wals }} +{{- end }} +{{- if .etcd.env.log_level }} +ETCD_LOG_LEVEL={{ .etcd.env.log_level }} +{{- end }} +{{- if .etcd.env.unsupported_arch }} +ETCD_UNSUPPORTED_ARCH={{ .etcd.env.unsupported_arch }} +{{- end }} + +# TLS settings +ETCD_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.crt +ETCD_CERT_FILE=/etc/ssl/etcd/ssl/server.crt +ETCD_KEY_FILE=/etc/ssl/etcd/ssl/server.key +ETCD_CLIENT_CERT_AUTH=true + +ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.crt +ETCD_PEER_CERT_FILE=/etc/ssl/etcd/ssl/server.crt +ETCD_PEER_KEY_FILE=/etc/ssl/etcd/ssl/server.key +ETCD_PEER_CLIENT_CERT_AUTH=true + +# CLI settings +ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 +ETCDCTL_CACERT=/etc/ssl/etcd/ssl/ca.crt +ETCDCTL_CERT=/etc/ssl/etcd/ssl/server.crt +ETCDCTL_KEY=/etc/ssl/etcd/ssl/server.key diff --git a/builtin/core/roles/image-registry/docker-compose/defaults/main.yaml b/builtin/core/roles/image-registry/docker-compose/defaults/main.yaml new file mode 100644 index 000000000..888b6f795 --- /dev/null +++ b/builtin/core/roles/image-registry/docker-compose/defaults/main.yaml @@ -0,0 +1,3 @@ +cri: + docker: + data_root: /var/lib/docker \ No newline at end of file diff --git a/builtin/core/roles/image-registry/docker-compose/files/containerd.service b/builtin/core/roles/image-registry/docker-compose/files/containerd.service new file mode 100644 index 000000000..5f67110ab --- /dev/null +++ b/builtin/core/roles/image-registry/docker-compose/files/containerd.service @@ -0,0 +1,26 @@ +[Unit] +Description=containerd container runtime +Documentation=https://containerd.io +After=network.target local-fs.target + +[Service] +ExecStartPre=-/sbin/modprobe overlay +ExecStart=/usr/local/bin/containerd + +Type=notify +Delegate=yes +KillMode=process +Restart=always +RestartSec=5 +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNPROC=infinity +LimitCORE=infinity +LimitNOFILE=1048576 +# Comment TasksMax if your systemd version does not supports it. +# Only systemd 226 and above support this version. +TasksMax=infinity +OOMScoreAdjust=-999 + +[Install] +WantedBy=multi-user.target diff --git a/builtin/core/roles/image-registry/docker-compose/files/docker.service b/builtin/core/roles/image-registry/docker-compose/files/docker.service new file mode 100644 index 000000000..929b8a436 --- /dev/null +++ b/builtin/core/roles/image-registry/docker-compose/files/docker.service @@ -0,0 +1,47 @@ +[Unit] +Description=Docker Application Container Engine +Documentation=https://docs.docker.com +# After=network-online.target firewalld.service containerd.service +# Wants=network-online.target +# Requires=docker.socket containerd.service + +[Service] +Type=notify +# the default is not to use systemd for cgroups because the delegate issues still +# exists and systemd currently does not support the cgroup feature set required +# for containers run by docker +ExecStart=/usr/local/bin/dockerd --containerd=/run/containerd/containerd.sock +ExecReload=/bin/kill -s HUP $MAINPID +TimeoutSec=0 +RestartSec=2 +Restart=always + +# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229. +# Both the old, and new location are accepted by systemd 229 and up, so using the old location +# to make them work for either version of systemd. +StartLimitBurst=3 + +# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230. +# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make +# this option work for either version of systemd. +StartLimitInterval=60s + +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity + +# Comment TasksMax if your systemd version does not support it. +# Only systemd 226 and above support this option. +TasksMax=infinity + +# set delegate yes so that systemd does not reset the cgroups of docker containers +Delegate=yes + +# kill only the docker process, not all processes in the cgroup +KillMode=process +OOMScoreAdjust=-500 + +[Install] +WantedBy=multi-user.target diff --git a/builtin/core/roles/image-registry/docker-compose/tasks/docker.yaml b/builtin/core/roles/image-registry/docker-compose/tasks/docker.yaml new file mode 100644 index 000000000..a5d3bf5e6 --- /dev/null +++ b/builtin/core/roles/image-registry/docker-compose/tasks/docker.yaml @@ -0,0 +1,34 @@ +--- +- name: Docker | Check if Docker is installed on the system + ignore_errors: true + command: docker --version + register: docker_install_version + +- name: Docker | Install and configure Docker if not present or version mismatch + when: or (.docker_install_version.error | empty | not) (.docker_install_version.stdout | hasPrefix (printf "Docker version %s," .cri.docker_version) | not) + block: + - name: Docker | Copy Docker binary archive to the remote node + copy: + src: >- + {{ .binary_dir }}/docker/{{ .cri.docker_version }}/{{ .binary_type }}/docker-{{ .cri.docker_version }}.tgz + dest: >- + {{ .tmp_dir }}/docker-{{ .cri.docker_version }}.tgz + - name: Docker | Generate Docker configuration file + template: + src: daemon.json + dest: /etc/docker/daemon.json + - name: Docker | Extract Docker binaries to /usr/local/bin + command: | + tar -C /usr/local/bin/ --strip-components=1 -xvf {{ .tmp_dir }}/docker-{{ .cri.docker_version }}.tgz --wildcards 'docker/*' + - name: Docker | Deploy the Docker systemd service file + copy: + src: docker.service + dest: /etc/systemd/system/docker.service + - name: Docker | Deploy the containerd systemd service file + copy: + src: containerd.service + dest: /etc/systemd/system/containerd.service + - name: Docker | Start and enable Docker and containerd services + command: | + systemctl daemon-reload && systemctl start containerd.service && systemctl enable containerd.service + systemctl daemon-reload && systemctl start docker.service && systemctl enable docker.service diff --git a/builtin/core/roles/image-registry/docker-compose/tasks/main.yaml b/builtin/core/roles/image-registry/docker-compose/tasks/main.yaml new file mode 100644 index 000000000..b40a6aacd --- /dev/null +++ b/builtin/core/roles/image-registry/docker-compose/tasks/main.yaml @@ -0,0 +1,15 @@ +--- +- include_tasks: docker.yaml + +- name: DockerCompose | Verify if Docker Compose is installed on the system + ignore_errors: true + command: docker-compose --version + register: dockercompose_install_version + +- name: DockerCompose | Install or update Docker Compose if not present or version mismatch + when: or (.dockercompose_install_version.error | empty | not) (.dockercompose_install_version.stdout | ne (printf "Docker Compose version %s" .image_registry.dockercompose_version)) + copy: + src: >- + {{ .binary_dir }}/image-registry/docker-compose/{{ .cri.dockercompose_version }}/{{ .binary_type }}/docker-compose + dest: /usr/local/bin/docker-compose + mode: 0755 \ No newline at end of file diff --git a/builtin/core/roles/image-registry/docker-compose/templates/daemon.json b/builtin/core/roles/image-registry/docker-compose/templates/daemon.json new file mode 100644 index 000000000..4a1248a9f --- /dev/null +++ b/builtin/core/roles/image-registry/docker-compose/templates/daemon.json @@ -0,0 +1,19 @@ +{ + "log-opts": { + "max-size": "5m", + "max-file":"3" + }, +{{- if .cri.docker.data_root | empty | not }} + "data-root": "{{ .cri.docker.data_root }}", +{{- end }} +{{- if .cri.registry.mirrors | empty | not }} + "registry-mirrors": {{ .cri.registry.mirrors | toJson }}, +{{- end }} + {{- if .cri.registry.insecure_registries | empty | not }} + "insecure-registries": {{ .cri.registry.insecure_registries | toJson }}, +{{- end }} + {{- if .cri.docker.bridge_ip | empty | not }} + "bip": "{{ .cri.docker.bridge_ip }}", +{{- end }} + "exec-opts": ["native.cgroupdriver={{ .cri.cgroup_driver | default "systemd" }}"] +} diff --git a/builtin/core/roles/image-registry/docker-registry/defaults/main.yaml b/builtin/core/roles/image-registry/docker-registry/defaults/main.yaml new file mode 100644 index 000000000..fc25713fc --- /dev/null +++ b/builtin/core/roles/image-registry/docker-registry/defaults/main.yaml @@ -0,0 +1,37 @@ +image_registry: + docker_registry: + version: 2 + config: + storage: nfs + nfs_dir: /share/registry + storage: + filesystem: + rootdir: /opt/docker-registry/data +# nfs_mount: /repository/registry # if set. will mount rootdirectory to nfs server in nfs_mount. +# azure: +# accountname: accountname +# accountkey: base64encodedaccountkey +# container: containername +# gcs: +# bucket: bucketname +# keyfile: /path/to/keyfile +# credentials: +# type: service_account +# project_id: project_id_string +# private_key_id: private_key_id_string +# private_key: private_key_string +# client_email: client@example.com +# client_id: client_id_string +# auth_uri: http://example.com/auth_uri +# token_uri: http://example.com/token_uri +# auth_provider_x509_cert_url: http://example.com/provider_cert_url +# client_x509_cert_url: http://example.com/client_cert_url +# rootdirectory: /gcs/object/name/prefix +# s3: +# accesskey: awsaccesskey +# secretkey: awssecretkey +# region: us-west-1 +# regionendpoint: http://myobjects.local +# bucket: bucketname +# keyid: mykeyid +# rootdirectory: /s3/object/name/prefix diff --git a/builtin/core/roles/image-registry/docker-registry/tasks/main.yaml b/builtin/core/roles/image-registry/docker-registry/tasks/main.yaml new file mode 100644 index 000000000..decb6e9b3 --- /dev/null +++ b/builtin/core/roles/image-registry/docker-registry/tasks/main.yaml @@ -0,0 +1,72 @@ +--- +- name: DockerRegistry | Synchronize registry image archive to remote host + copy: + src: >- + {{ .binary_dir }}/image-registry/docker-registry/{{ .docker_registry_version }}/{{ .binary_type }}/docker-registry-{{ .docker_registry_version }}-linux-{{ .binary_type }}.tgz + dest: >- + /opt/docker-registry/{{ .docker_registry_version }}/docker-registry-{{ .docker_registry_version }}-linux-{{ .binary_type }}.tgz + +- name: DockerRegistry | Ensure NFS directory is mounted for registry storage + command: | + {{- if .os.release.ID_LIKE | eq "debian" }} + yum update && yum install -y nfs-utils + {{- else if .os.release.ID_LIKE | unquote | eq "rhel fedora" }} + apt update && apt install -y nfs-common + {{- end }} + {{- if }} + {{- $internalIPv4 := index .hostvars (.groups.nfs | default list | first) "internal_ipv4" | default "" }} + {{- $internalIPv6 := index .hostvars (.groups.nfs | default list | first) "internal_ipv6" | default "" }} + {{- if $internalIPv4 | empty | not }} + mount -t nfs {{ $internalIPv4 }}:{{ .image_registry.docker_registry.storage.filesystem.nfs_mount }} {{ .image_registry.docker_registry.storage.filesystem.rootdir }} + {{- else if ne $internalIPv6 "" }} + {{ $internalIPv6 | empty | not }} + mount -t nfs {{ $internalIPv6 }}:{{ .image_registry.docker_registry.storage.filesystem.nfs_mount }} {{ .image_registry.docker_registry.storage.filesystem.rootdir }} + {{- end }} + when: + - .image_registry.docker_registry.storage.filesystem.nfs_mount | empty | not + - .groups.nfs | default list | len | eq 1 + +- name: DockerRegistry | Load registry image into Docker + command: | + docker load -i /opt/docker-registry/{{ .docker_registry_version }}/docker-registry-{{ .docker_registry_version }}-linux-{{ .binary_type }}.tgz + +- name: DockerRegistry | Synchronize registry certificate to remote host + copy: + src: >- + {{ .binary_dir }}/pki/image_registry.crt + dest: >- + /opt/docker-registry/{{ .docker_registry_version }}/ssl/server.crt + +- name: DockerRegistry | Synchronize registry private key to remote host + copy: + src: >- + {{ .binary_dir }}/pki/image_registry.key + dest: >- + /opt/docker-registry/{{ .docker_registry_version }}/ssl/server.key + +- name: DockerRegistry | Generate Docker Compose file for registry + template: + src: docker-compose.yaml + dest: >- + /opt/docker-registry/{{ .docker_registry_version }}/docker-compose.yml + +- name: DockerRegistry | Generate registry configuration file + template: + src: config.yaml + dest: >- + /opt/docker-registry/{{ .docker_registry_version }}/config.yml + +- name: DockerRegistry | Register registry as a systemd service + template: + src: docker-registry.service + dest: /etc/systemd/system/docker-registry.service + +- name: DockerRegistry | Start and enable registry systemd service + command: systemctl daemon-reload && systemctl start docker-registry.service && systemctl enable docker-registry.service + +- name: DockerRegistry | Wait for registry service to become available + command: | + if ! timeout 300 bash -c 'while ! nc -zv localhost 443; do sleep 2; done'; then + echo "ERROR: Docker Registry did not start within 5 minutes!" + exit 1 + fi diff --git a/builtin/core/roles/image-registry/docker-registry/templates/config.yaml b/builtin/core/roles/image-registry/docker-registry/templates/config.yaml new file mode 100644 index 000000000..bf8dfe665 --- /dev/null +++ b/builtin/core/roles/image-registry/docker-registry/templates/config.yaml @@ -0,0 +1,218 @@ +version: 0.1 +log: + accesslog: + disabled: true + level: info + formatter: text + fields: + service: registry + environment: staging +# hooks: +# - type: mail +# disabled: true +# levels: +# - panic +# options: +# smtp: +# addr: mail.example.com:25 +# username: mailuser +# password: password +# insecure: true +# from: sender@example.com +# to: +# - errors@example.com +storage: +{{- if .image_registry.docker_registry.storage.filesystem.rootdir | empty | not }} + filesystem: + rootdirectory: {{ .image_registry.docker_registry.storage.filesystem.rootdir }} + maxthreads: 100 +{{- end }} +{{- if .image_registry.docker_registry.storage.azure }} + azure: + accountname: {{ .image_registry.docker_registry.storage.azure.accountname }} + accountkey: {{ .image_registry.docker_registry.storage.azure.accountkey }} + container: {{ .image_registry.docker_registry.storage.azure.container }} +{{- end }} +{{- if .image_registry.docker_registry.storage.gcs | empty | not }} + gcs: + bucket: {{ .image_registry.docker_registry.storage.gcs.bucket }} + keyfile: {{ .image_registry.docker_registry.storage.gcs.keyfile }} + credentials: + type: service_account + project_id: {{ .image_registry.docker_registry.storage.gcs.credentials.project_id }} + private_key_id: {{ .image_registry.docker_registry.storage.gcs.credentials.private_key_id }} + private_key: {{ .image_registry.docker_registry.storage.gcs.credentials.private_key }} + client_email: {{ .image_registry.docker_registry.storage.gcs.credentials.client_email }} + client_id: {{ .image_registry.docker_registry.storage.gcs.credentials.client_id }} + auth_uri: {{ .image_registry.docker_registry.storage.gcs.credentials.auth_uri }} + token_uri: {{ .image_registry.docker_registry.storage.gcs.credentials.token_uri }} + auth_provider_x509_cert_url: {{ .image_registry.docker_registry.storage.gcs.credentials.auth_provider_x509_cert_url }} + client_x509_cert_url: {{ .image_registry.docker_registry.storage.gcs.credentials.client_x509_cert_url }} + rootdirectory: {{ .image_registry.docker_registry.storage.gcs.rootdirectory }} +{{- end }} +{{- if .image_registry.docker_registry.storage.s3 | empty | not }} + s3: + accesskey: {{ .image_registry.docker_registry.storage.s3.accesskey }} + secretkey: {{ .image_registry.docker_registry.storage.s3.secretkey }} + region: {{ .image_registry.docker_registry.storage.s3.region }} + regionendpoint: {{ .image_registry.docker_registry.storage.s3.regionendpoint }} + forcepathstyle: true + accelerate: false + bucket: {{ .image_registry.docker_registry.storage.s3.bucket }} + encrypt: true + keyid: {{ .image_registry.docker_registry.storage.s3.keyid }} + secure: true + v4auth: true + chunksize: 5242880 + multipartcopychunksize: 33554432 + multipartcopymaxconcurrency: 100 + multipartcopythresholdsize: 33554432 + rootdirectory: {{ .image_registry.docker_registry.storage.s3.rootdirectory }} + usedualstack: false + loglevel: debug +{{- end }} +# inmemory: # This driver takes no parameters + delete: + enabled: false + redirect: + disable: false + cache: + blobdescriptor: inmemory + blobdescriptorsize: 10000 + maintenance: + uploadpurging: + enabled: true + age: 168h + interval: 24h + dryrun: false + readonly: + enabled: false +#auth: +# silly: +# realm: silly-realm +# service: silly-service +# token: +# autoredirect: true +# realm: token-realm +# service: token-service +# issuer: registry-token-issuer +# rootcertbundle: /root/certs/bundle +# htpasswd: +# realm: basic-realm +# path: /path/to/htpasswd +#middleware: +# registry: +# - name: ARegistryMiddleware +# options: +# foo: bar +# repository: +# - name: ARepositoryMiddleware +# options: +# foo: bar +# storage: +# - name: cloudfront +# options: +# baseurl: https://my.cloudfronted.domain.com/ +# privatekey: /path/to/pem +# keypairid: cloudfrontkeypairid +# duration: 3000s +# ipfilteredby: awsregion +# awsregion: us-east-1, use-east-2 +# updatefrequency: 12h +# iprangesurl: https://ip-ranges.amazonaws.com/ip-ranges.json +# - name: redirect +# options: +# baseurl: https://example.com/ +http: + addr: 0.0.0.0:5000 +# prefix: /my/nested/registry/ +# host: https://myregistryaddress.org:5000 + secret: asecretforlocaldevelopment + relativeurls: false + draintimeout: 60s + tls: + certificate: /etc/registry/ssl/server.crt + key: /etc/registry/ssl/server.key +# clientcas: +# - /path/to/ca.pem +# - /path/to/another/ca.pem +# letsencrypt: +# cachefile: /path/to/cache-file +# email: emailused@letsencrypt.com +# hosts: [myregistryaddress.org] +# directoryurl: https://acme-v02.api.letsencrypt.org/directory +# debug: +# addr: localhost:5001 +# prometheus: +# enabled: true +# path: /metrics + headers: + X-Content-Type-Options: [nosniff] + http2: + disabled: false + h2c: + enabled: false +#notifications: +# events: +# includereferences: true +# endpoints: +# - name: alistener +# disabled: false +# url: https://my.listener.com/event +# headers: +# timeout: 1s +# threshold: 10 +# backoff: 1s +# ignoredmediatypes: +# - application/octet-stream +# ignore: +# mediatypes: +# - application/octet-stream +# actions: +# - pull +#redis: +# addr: localhost:6379 +# password: asecret +# db: 0 +# dialtimeout: 10ms +# readtimeout: 10ms +# writetimeout: 10ms +# pool: +# maxidle: 16 +# maxactive: 64 +# idletimeout: 300s +# tls: +# enabled: false +health: + storagedriver: + enabled: true + interval: 10s + threshold: 3 +# file: +# - file: /path/to/checked/file +# interval: 10s +# http: +# - uri: http://server.to.check/must/return/200 +# headers: +# Authorization: [Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==] +# statuscode: 200 +# timeout: 3s +# interval: 10s +# threshold: 3 +# tcp: +# - addr: redis-server.domain.com:6379 +# timeout: 3s +# interval: 10s +## threshold: 3 +#proxy: +# remoteurl: https://registry-1.docker.io +# username: [username] +# password: [password] +# ttl: 168h +#validation: +# manifests: +# urls: +# allow: +# - ^https?://([^/]+\.)*example\.com/ +# deny: +# - ^https?://www\.example\.com/ diff --git a/builtin/core/roles/image-registry/docker-registry/templates/docker-compose.yaml b/builtin/core/roles/image-registry/docker-registry/templates/docker-compose.yaml new file mode 100644 index 000000000..ddac10b92 --- /dev/null +++ b/builtin/core/roles/image-registry/docker-registry/templates/docker-compose.yaml @@ -0,0 +1,56 @@ +--- +version: '2.3' +services: + registry: + image: registry:{{ .docker_registry_version }} + container_name: registry + restart: always + dns_search: . + cap_drop: + - ALL + cap_add: + - CHOWN + - DAC_OVERRIDE + - SETGID + - SETUID + volumes: + - type: bind + source: /opt/docker-registry/{{ .docker_registry_version }}/ssl/ + target: /etc/registry/ssl/ + - type: bind + source: /opt/docker-registry/{{ .docker_registry_version }}/config.yml + target: /etc/docker/registry/config.yml + ports: + - 443:5000 + networks: + - registry +{{- if .image_registry.ha_vip | empty | not }} + keepalived: + image: osixia/keepalived:{{ .keepalived_version }} + container_name: keepalived + command: + - --copy-service + network_mode: "host" + restart: always + dns_search: . + cap_drop: + - ALL + cap_add: + - NET_ADMIN + - NET_BROADCAST + - NET_RAW + - CHOWN + - DAC_OVERRIDE + depends_on: + - registry + volumes: + - type: bind + source: /opt/keepalived/{{ .keepalived_version }}/keepalived.conf + target: /container/service/keepalived/assets/keepalived.conf + - type: bind + source: /opt/keepalived/{{ .keepalived_version }}/healthcheck.sh + target: /etc/keepalived/healthcheck.sh +{{- end }} +networks: + registry: + external: false diff --git a/builtin/core/roles/image-registry/docker-registry/templates/docker-registry.service b/builtin/core/roles/image-registry/docker-registry/templates/docker-registry.service new file mode 100644 index 000000000..5a51fac44 --- /dev/null +++ b/builtin/core/roles/image-registry/docker-registry/templates/docker-registry.service @@ -0,0 +1,12 @@ +[Unit] +Description=docker-registry +After=docker.service systemd-networkd.service systemd-resolved.service +Requires=docker.service + +[Service] +Type=simple +ExecStart=/usr/local/bin/docker-compose -p registry -f /opt/docker-registry/{{ .docker_registry_version }}/docker-compose.yml up +ExecStop=/usr/local/bin/docker-compose -p registry down +Restart=on-failure +[Install] +WantedBy=multi-user.target diff --git a/builtin/core/roles/image-registry/harbor/defaults/main.yaml b/builtin/core/roles/image-registry/harbor/defaults/main.yaml new file mode 100644 index 000000000..a6b429c71 --- /dev/null +++ b/builtin/core/roles/image-registry/harbor/defaults/main.yaml @@ -0,0 +1,4 @@ +image_registry: + # Virtual IP address for repository High Availability. the Virtual IP address should be available. + harbor: + data_dir: /opt/harbor/data \ No newline at end of file diff --git a/builtin/core/roles/image-registry/harbor/tasks/main.yaml b/builtin/core/roles/image-registry/harbor/tasks/main.yaml new file mode 100644 index 000000000..8575baf9b --- /dev/null +++ b/builtin/core/roles/image-registry/harbor/tasks/main.yaml @@ -0,0 +1,101 @@ +--- +- name: Harbor | Synchronize Harbor offline package to remote host + copy: + src: >- + {{ .binary_dir }}/image-registry/harbor/{{ .image_registry.harbor_version }}/{{ .binary_type }}/harbor-offline-installer-{{ .image_registry.harbor_version }}.tgz + dest: >- + /opt/harbor/{{ .image_registry.harbor_version }}/harbor-offline-installer-{{ .image_registry.harbor_version }}.tgz + +- name: Harbor | Extract Harbor offline package + command: | + cd /opt/harbor/{{ .image_registry.harbor_version }}/ && tar -zxvf harbor-offline-installer-{{ .image_registry.harbor_version }}.tgz + +- name: Harbor | Synchronize image registry certificate to remote host + copy: + src: >- + {{ .binary_dir }}/pki/image_registry.crt + dest: >- + /opt/harbor/{{ .image_registry.harbor_version }}/ssl/server.crt + +- name: Harbor | Synchronize image registry private key to remote host + copy: + src: >- + {{ .binary_dir }}/pki/image_registry.key + dest: >- + /opt/harbor/{{ .image_registry.harbor_version }}/ssl/server.key + +- name: Harbor | Generate Harbor configuration file + template: + src: harbor.yml + dest: >- + /opt/harbor/{{ .image_registry.harbor_version }}/harbor/harbor.yml + +- name: Harbor | Install Harbor registry + command: | + cd /opt/harbor/{{ .image_registry.harbor_version }}/harbor && /bin/bash install.sh + +- name: Harbor | Register Harbor as a systemd service + template: + src: harbor.service + dest: /etc/systemd/system/harbor.service + +- name: Harbor | Start and enable Harbor service + command: systemctl daemon-reload && systemctl start harbor.service && systemctl enable harbor.service + +- name: Harbor | Configure HA and synchronize Harbor images + when: + - .image_registry.ha_vip | empty | not + - .groups.image_registry | len | lt 1 + block: + - name: Harbor | Add keepalived service to Harbor docker-compose + command: | + KEEPALIVED_SERVICE='# keepalived is generated by kubekey. + keepalived: + image: osixia/keepalived:{{ .keepalived_version }} + container_name: keepalived + command: + - --copy-service + network_mode: "host" + restart: always + dns_search: . + cap_drop: + - ALL + cap_add: + - NET_ADMIN + - NET_BROADCAST + - NET_RAW + - CHOWN + - DAC_OVERRIDE + depends_on: + - proxy + volumes: + - type: bind + source: /opt/keepalived/{{ .keepalived_version }}/keepalived.conf + target: /container/service/keepalived/assets/keepalived.conf + - type: bind + source: /opt/keepalived/{{ .keepalived_version }}/healthcheck.sh + target: /etc/keepalived/healthcheck.sh' + TARGET_FILE="/opt/harbor/{{ .image_registry.harbor_version }}/harbor/docker-compose.yml" + TMP_FILE="/opt/harbor/{{ .image_registry.harbor_version }}/harbor/docker-compose.yml.tmp" + awk -v service="$KEEPALIVED_SERVICE" ' + /^services:/ { + print + print service + next + } + { print } + ' "$TARGET_FILE" > "$TMP_FILE" && mv "$TMP_FILE" "$TARGET_FILE" + systemctl restart harbor.service + - name: Harbor | Wait for Harbor service to become ready + command: | + if ! timeout 300 bash -c 'while ! nc -zv localhost 443; do sleep 2; done'; then + echo "ERROR: Harbor did not start within 5 minutes!" + exit 1 + fi + - name: Harbor | Synchronize harbor-replications script to remote host + template: + src: harbor-replications.sh + dest: /opt/harbor/scripts/harbor-replications.sh + mode: 0755 + - name: Harbor | Execute harbor-replications script + command: bash /opt/harbor/scripts/harbor-replications.sh \ No newline at end of file diff --git a/builtin/core/roles/image-registry/harbor/templates/harbor-replications.sh b/builtin/core/roles/image-registry/harbor/templates/harbor-replications.sh new file mode 100644 index 000000000..5afa78d23 --- /dev/null +++ b/builtin/core/roles/image-registry/harbor/templates/harbor-replications.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +function createRegistries() { +{{- range .groups.image_registry | default list }} + {{- if ne . $.inventory_hostname }} + curl -k -u '{{ printf "%s:%s" $.image_registry.auth.username $.image_registry.auth.password }}' -X POST -H "Content-Type: application/json" "https://{{ $.inventory_hostname }}/api/v2.0/registries" -d "{\"name\": \"{{ . }}\", \"type\": \"harbor\", \"url\":\"https://{{ . }}:7443\", \"credential\": {\"access_key\": \"{{ $.image_registry.auth.username }}\", \"access_secret\": \"{{ $.image_registry.auth.password }}\"}, \"insecure\": true}" + {{- end }} +{{- end }} +} + +function createReplication() { +{{- range $index, $host := .groups.image_registry | default list }} + {{- if ne $host $.inventory_hostname }} + curl -k -u '{{ printf "%s:%s" $.image_registry.auth.username $.image_registry.auth.password }}' -X POST -H "Content-Type: application/json" "https://{{ $.inventory_hostname }}/api/v2.0/replication/policies" -d "{\"name\": \"{{ printf "%s_%s" $.inventory_hostname $host }}\", \"enabled\": true, \"deletion\":true, \"override\":true, \"replicate_deletion\":true, \"dest_registry\":{ \"id\": {{ $index }}, \"name\": \"{{ $host }}\"}, \"trigger\": {\"type\": \"event_based\"}, \"dest_namespace_replace_count\":1 }" + {{- end }} +{{- end }} +} + +createRegistries +createReplication diff --git a/builtin/core/roles/image-registry/harbor/templates/harbor.service b/builtin/core/roles/image-registry/harbor/templates/harbor.service new file mode 100644 index 000000000..969840f9e --- /dev/null +++ b/builtin/core/roles/image-registry/harbor/templates/harbor.service @@ -0,0 +1,12 @@ +[Unit] +Description=harbor +After=docker.service systemd-networkd.service systemd-resolved.service +Requires=docker.service + +[Service] +Type=simple +ExecStart=/usr/local/bin/docker-compose -p harbor -f /opt/harbor/{{ .image_registry.harbor_version }}/harbor/docker-compose.yml up +ExecStop=/usr/local/bin/docker-compose -p harbor down +Restart=on-failure +[Install] +WantedBy=multi-user.target diff --git a/builtin/core/roles/image-registry/harbor/templates/harbor.yml b/builtin/core/roles/image-registry/harbor/templates/harbor.yml new file mode 100644 index 000000000..ad68b2fc4 --- /dev/null +++ b/builtin/core/roles/image-registry/harbor/templates/harbor.yml @@ -0,0 +1,311 @@ +# Configuration file of Harbor + +# The IP address or hostname to access admin UI and registry service. +# DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients. +hostname: {{ .image_registry.auth.registry }} + +# http related config +http: + # port for http, default is 80. If https enabled, this port will redirect to https port + port: 80 + +# https related config +https: + # https port for harbor, default is 443 + port: 443 + # The path of cert and key files for nginx + certificate: /opt/harbor/{{ .image_registry.harbor_version }}/ssl/server.crt + private_key: /opt/harbor/{{ .image_registry.harbor_version }}/ssl/server.key + # enable strong ssl ciphers (default: false) + # strong_ssl_ciphers: false + +# # Uncomment following will enable tls communication between all harbor components +# internal_tls: +# # set enabled to true means internal tls is enabled +# enabled: true +# # put your cert and key files on dir +# dir: /etc/harbor/tls/internal + + +# Uncomment external_url if you want to enable external proxy +# And when it enabled the hostname will no longer used +# external_url: https://reg.mydomain.com:8433 + +# The initial password of Harbor admin +# It only works in first time to install harbor +# Remember Change the admin password from UI after launching Harbor. +harbor_admin_password: {{ .image_registry.auth.password }} + +# Harbor DB configuration +database: + # The password for the root user of Harbor DB. Change this before any production use. + password: root123 + # The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained. + max_idle_conns: 100 + # The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections. + # Note: the default number of connections is 1024 for postgres of harbor. + max_open_conns: 900 + # The maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If it <= 0, connections are not closed due to a connection's age. + # The value is a duration string. A duration string is a possibly signed sequence of decimal numbers, each with optional fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + conn_max_lifetime: 5m + # The maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If it <= 0, connections are not closed due to a connection's idle time. + # The value is a duration string. A duration string is a possibly signed sequence of decimal numbers, each with optional fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + conn_max_idle_time: 0 + +# The default data volume +data_volume: {{ .image_registry.harbor.data_dir }} + +# Harbor Storage settings by default is using /data dir on local filesystem +# Uncomment storage_service setting If you want to using external storage +# storage_service: +# # ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore +# # of registry's containers. This is usually needed when the user hosts a internal storage with self signed certificate. +# ca_bundle: + +# # storage backend, default is filesystem, options include filesystem, azure, gcs, s3, swift and oss +# # for more info about this configuration please refer https://docs.docker.com/registry/configuration/ +# filesystem: +# maxthreads: 100 +# # set disable to true when you want to disable registry redirect +# redirect: +# disable: false + +# Trivy configuration +# +# Trivy DB contains vulnerability information from NVD, Red Hat, and many other upstream vulnerability databases. +# It is downloaded by Trivy from the GitHub release page https://github.com/aquasecurity/trivy-db/releases and cached +# in the local file system. In addition, the database contains the update timestamp so Trivy can detect whether it +# should download a newer version from the Internet or use the cached one. Currently, the database is updated every +# 12 hours and published as a new release to GitHub. +trivy: + # ignoreUnfixed The flag to display only fixed vulnerabilities + ignore_unfixed: false + # skipUpdate The flag to enable or disable Trivy DB downloads from GitHub + # + # You might want to enable this flag in test or CI/CD environments to avoid GitHub rate limiting issues. + # If the flag is enabled you have to download the `trivy-offline.tar.gz` archive manually, extract `trivy.db` and + # `metadata.json` files and mount them in the `/home/scanner/.cache/trivy/db` path. + skip_update: false + # + # skipJavaDBUpdate If the flag is enabled you have to manually download the `trivy-java.db` file and mount it in the + # `/home/scanner/.cache/trivy/java-db/trivy-java.db` path + skip_java_db_update: false + # + # The offline_scan option prevents Trivy from sending API requests to identify dependencies. + # Scanning JAR files and pom.xml may require Internet access for better detection, but this option tries to avoid it. + # For example, the offline mode will not try to resolve transitive dependencies in pom.xml when the dependency doesn't + # exist in the local repositories. It means a number of detected vulnerabilities might be fewer in offline mode. + # It would work if all the dependencies are in local. + # This option doesn't affect DB download. You need to specify "skip-update" as well as "offline-scan" in an air-gapped environment. + offline_scan: false + # + # Comma-separated list of what security issues to detect. Possible values are `vuln`, `config` and `secret`. Defaults to `vuln`. + security_check: vuln + # + # insecure The flag to skip verifying registry certificate + insecure: false + # github_token The GitHub access token to download Trivy DB + # + # Anonymous downloads from GitHub are subject to the limit of 60 requests per hour. Normally such rate limit is enough + # for production operations. If, for any reason, it's not enough, you could increase the rate limit to 5000 + # requests per hour by specifying the GitHub access token. For more details on GitHub rate limiting please consult + # https://docs.github.com/rest/overview/resources-in-the-rest-api#rate-limiting + # + # You can create a GitHub token by following the instructions in + # https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line + # + # github_token: xxx + +jobservice: + # Maximum number of job workers in job service + max_job_workers: 10 + # The jobLoggers backend name, only support "STD_OUTPUT", "FILE" and/or "DB" + job_loggers: + - STD_OUTPUT + - FILE + # - DB + # The jobLogger sweeper duration (ignored if `jobLogger` is `stdout`) + logger_sweeper_duration: 1 #days + +notification: + # Maximum retry count for webhook job + webhook_job_max_retry: 3 + # HTTP client timeout for webhook job + webhook_job_http_client_timeout: 3 #seconds + +# Log configurations +log: + # options are debug, info, warning, error, fatal + level: info + # configs for logs in local storage + local: + # Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated. + rotate_count: 50 + # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. + # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G + # are all valid. + rotate_size: 200M + # The directory on your host that store log + location: /var/log/harbor + + # Uncomment following lines to enable external syslog endpoint. + # external_endpoint: + # # protocol used to transmit log to external endpoint, options is tcp or udp + # protocol: tcp + # # The host of external endpoint + # host: localhost + # # Port of external endpoint + # port: 5140 + +#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY! +_version: 2.10.0 + +# Uncomment external_database if using external database. +# external_database: +# harbor: +# host: harbor_db_host +# port: harbor_db_port +# db_name: harbor_db_name +# username: harbor_db_username +# password: harbor_db_password +# ssl_mode: disable +# max_idle_conns: 2 +# max_open_conns: 0 + +# Uncomment redis if need to customize redis db +# redis: +# # db_index 0 is for core, it's unchangeable +# # registry_db_index: 1 +# # jobservice_db_index: 2 +# # trivy_db_index: 5 +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_db_index: 7 + +# Uncomment redis if need to customize redis db +# redis: +# # db_index 0 is for core, it's unchangeable +# # registry_db_index: 1 +# # jobservice_db_index: 2 +# # trivy_db_index: 5 +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 + +# Uncomment external_redis if using external Redis server +# external_redis: +# # support redis, redis+sentinel +# # host for redis: : +# # host for redis+sentinel: +# # :,:,: +# host: redis:6379 +# password: +# # Redis AUTH command was extended in Redis 6, it is possible to use it in the two-arguments AUTH form. +# # there's a known issue when using external redis username ref:https://github.com/goharbor/harbor/issues/18892 +# # if you care about the image pull/push performance, please refer to this https://github.com/goharbor/harbor/wiki/Harbor-FAQs#external-redis-username-password-usage +# # username: +# # sentinel_master_set must be set to support redis+sentinel +# #sentinel_master_set: +# # db_index 0 is for core, it's unchangeable +# registry_db_index: 1 +# jobservice_db_index: 2 +# trivy_db_index: 5 +# idle_timeout_seconds: 30 +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 + +# Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert. +# uaa: +# ca_file: /path/to/ca + +# Global proxy +# Config http proxy for components, e.g. http://my.proxy.com:3128 +# Components doesn't need to connectorVars to each others via http proxy. +# Remove component from `components` array if want disable proxy +# for it. If you want use proxy for replication, MUST enable proxy +# for core and jobservice, and set `http_proxy` and `https_proxy`. +# Add domain to the `no_proxy` field, when you want disable proxy +# for some special registry. +proxy: + http_proxy: + https_proxy: + no_proxy: + components: + - core + - jobservice + - trivy + +# metric: +# enabled: false +# port: 9090 +# path: /metrics + +# Trace related config +# only can enable one trace provider(jaeger or otel) at the same time, +# and when using jaeger as provider, can only enable it with agent mode or collector mode. +# if using jaeger collector mode, uncomment endpoint and uncomment username, password if needed +# if using jaeger agetn mode uncomment agent_host and agent_port +# trace: +# enabled: true +# # set sample_rate to 1 if you wanna sampling 100% of trace data; set 0.5 if you wanna sampling 50% of trace data, and so forth +# sample_rate: 1 +# # # namespace used to differenciate different harbor services +# # namespace: +# # # attributes is a key value dict contains user defined attributes used to initialize trace provider +# # attributes: +# # application: harbor +# # # jaeger should be 1.26 or newer. +# # jaeger: +# # endpoint: http://hostname:14268/api/traces +# # username: +# # password: +# # agent_host: hostname +# # # export trace data by jaeger.thrift in compact mode +# # agent_port: 6831 +# # otel: +# # endpoint: hostname:4318 +# # url_path: /v1/traces +# # compression: false +# # insecure: true +# # # timeout is in seconds +# # timeout: 10 + +# Enable purge _upload directories +upload_purging: + enabled: true + # remove files in _upload directories which exist for a period of time, default is one week. + age: 168h + # the interval of the purge operations + interval: 24h + dryrun: false + +# Cache layer configurations +# If this feature enabled, harbor will cache the resource +# `project/project_metadata/repository/artifact/manifest` in the redis +# which can especially help to improve the performance of high concurrent +# manifest pulling. +# NOTICE +# If you are deploying Harbor in HA mode, make sure that all the harbor +# instances have the same behaviour, all with caching enabled or disabled, +# otherwise it can lead to potential data inconsistency. +cache: + # not enabled by default + enabled: false + # keep cache for one day by default + expire_hours: 24 + +# Harbor core configurations +# Uncomment to enable the following harbor core related configuration items. +# core: +# # The provider for updating project quota(usage), there are 2 options, redis or db, +# # by default is implemented by db but you can switch the updation via redis which +# # can improve the performance of high concurrent pushing to the same project, +# # and reduce the database connections spike and occupies. +# # By redis will bring up some delay for quota usage updation for display, so only +# # suggest switch provider to redis if you were ran into the db connections spike aroud +# # the scenario of high concurrent pushing to same project, no improvment for other scenes. +# quota_update_provider: redis # Or db diff --git a/builtin/core/roles/image-registry/keepalived/files/healthcheck.sh b/builtin/core/roles/image-registry/keepalived/files/healthcheck.sh new file mode 100644 index 000000000..43bd99d6b --- /dev/null +++ b/builtin/core/roles/image-registry/keepalived/files/healthcheck.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +nc -zv -w 2 localhost 443 > /dev/null 2>&1 + +if [ $? -eq 0 ]; then + exit 0 +else + exit 1 +fi diff --git a/builtin/core/roles/image-registry/keepalived/tasks/main.yaml b/builtin/core/roles/image-registry/keepalived/tasks/main.yaml new file mode 100644 index 000000000..1ef1db441 --- /dev/null +++ b/builtin/core/roles/image-registry/keepalived/tasks/main.yaml @@ -0,0 +1,65 @@ +--- +- name: Keepalived | Discover network interface for HA VIP + block: + - name: Keepalived | Gather all network interfaces with CIDR addresses + command: | + ip -o addr show | awk ' + BEGIN { + printf "[\n"; + first = 1; + } + /inet / && !/ lo|docker|br-|veth/ { + if (!first) { + printf ",\n"; + } + first = 0; + printf " {\n"; + printf " \"interface\": \"%s\",\n", $2; + printf " \"cidr\": \"%s\"\n", $4; + printf " }"; + } + END { + printf "\n]\n"; + } + ' + register: interface + register_type: json + - name: Keepalived | Select interface matching HA VIP + set_fact: + ha_vip_interface: >- + {{- $interface := "" }} + {{- range .interface.stdout | default list -}} + {{- if .cidr | ipInCIDR | has $.image_registry.ha_vip -}} + {{- $interface = .interface -}} + {{- end -}} + {{- end -}} + {{ $interface }} + - name: Keepalived | Ensure matching network interface exists + assert: + that: .kube_vip_interface | empty + fail_msg: "Cannot find a network interface that matches the HA VIP." + +- name: Keepalived | Synchronize keepalived image to remote host + copy: + src: >- + {{ .binary_dir }}/image-registry/keepalived/{{ .keepalived_version }}/{{ .binary_type }}/keepalived-{{ .keepalived_version }}-linux-{{ .binary_type }}.tgz + dest: >- + /opt/keepalived/{{ .keepalived_version }}/keepalived-{{ .keepalived_version }}-linux-{{ .binary_type }}.tgz + +- name: Keepalived | Load keepalived image into Docker + command: | + docker load -i /opt/keepalived/{{ .keepalived_version }}/keepalived-{{ .keepalived_version }}-linux-{{ .binary_type }}.tgz + +- name: Keepalived | Synchronize keepalived configuration file to remote host + template: + src: keepalived.conf + dest: >- + /opt/keepalived/{{ .keepalived_version }}/keepalived.conf + mode: 0664 + +- name: Keepalived | Synchronize healthcheck script to remote host + copy: + src: healthcheck.sh + dest: >- + /opt/keepalived/{{ .keepalived_version }}/healthcheck.sh + mode: 0755 diff --git a/builtin/core/roles/image-registry/keepalived/templates/keepalived.conf b/builtin/core/roles/image-registry/keepalived/templates/keepalived.conf new file mode 100644 index 000000000..e160c6208 --- /dev/null +++ b/builtin/core/roles/image-registry/keepalived/templates/keepalived.conf @@ -0,0 +1,30 @@ +vrrp_script healthcheck { + script "/etc/keepalived/healthcheck.sh" + interval 10 + fall 2 + rise 2 + timeout 5 + init_fail +} + global_defs { + script_user root + router_id harbor-ha + enable_script_security +} + vrrp_instance VI_1 { + state {{ if .groups.image_registry | first | eq .inventory_hostname }}MASTER{{ else }}BACKUP{{ end }} + interface {{ .ha_vip_interface }} + virtual_router_id 31 + priority 50 + advert_int 1 + authentication { + auth_type PASS + auth_pass k8s-test + } + virtual_ipaddress { + {{ .image_registry.ha_vip }} + } + track_script { + healthcheck + } + } diff --git a/builtin/core/roles/image-registry/meta/main.yaml b/builtin/core/roles/image-registry/meta/main.yaml new file mode 100644 index 000000000..4782dfa1e --- /dev/null +++ b/builtin/core/roles/image-registry/meta/main.yaml @@ -0,0 +1,15 @@ +--- +dependencies: + - role: image-registry/docker-compose + + - role: image-registry/keepalived + when: + - .image_registry.ha_vip | empty | not + - .groups.image_registry | len | lt 1 + + - role: image-registry/harbor + when: .image_registry.type | eq "harbor" + + - role: image-registry/docker-registry + when: .image_registry.type | eq "docker-registry" + diff --git a/builtin/core/roles/image-registry/tasks/main.yaml b/builtin/core/roles/image-registry/tasks/main.yaml new file mode 100644 index 000000000..ad5b93d0e --- /dev/null +++ b/builtin/core/roles/image-registry/tasks/main.yaml @@ -0,0 +1,51 @@ +--- +- name: ImageRegistry | Synchronize images to remote host + when: + - .image_manifests | default list | empty | not + - .download.download_image + copy: + src: >- + {{ .binary_dir }}/images/ + dest: >- + {{ .image_registry.images_dir }} + +- name: ImageRegistry | Ensure Harbor project exists for each image + when: .image_registry.type | eq "harbor" + command: | + # Traverse first-level subdirectories in images_dir, skipping 'blobs' + for registry_dir in {{ .image_registry.images_dir }}*; do + if [ ! -d "$registry_dir" ] || [ "$(basename "$registry_dir")" = "blobs" ]; then + continue + fi + + # Traverse second-level subdirectories in each registry_dir + for project_dir in "$registry_dir"/*; do + if [ ! -d "$project_dir" ]; then + continue + fi + + project=$(basename "$project_dir") + + # Check if the Harbor project exists; create it if it does not + resp=$(curl -u "{{ .image_registry.auth.username }}:{{ .image_registry.auth.password }}" -k -X GET "https://{{ .image_registry.auth.registry }}/api/v2.0/projects/${project}") + if echo "$resp" | grep -q '"code":"NOT_FOUND"'; then + curl -u "{{ .image_registry.auth.username }}:{{ .image_registry.auth.password }}" -k -X POST \ + -H "Content-Type: application/json" \ + "https://{{ .image_registry.auth.registry }}/api/v2.0/projects" \ + -d "{ \"project_name\": \"${project}\", \"public\": true}" + fi + done + done + +- name: ImageRegistry | Push images package to image registry + image: + push: + images_dir: >- + {{ .image_registry.images_dir }} + dest: >- + {{ .image_registry.auth.registry }}/{{ .module.image.src.reference.repository }}:{{ .module.image.src.reference.reference }} + username: >- + {{ .image_registry.auth.username }} + password: >- + {{ .image_registry.auth.password }} + skip_tls_verify: true diff --git a/builtin/core/roles/kubernetes/certs/files/k8s-certs-renew.service b/builtin/core/roles/kubernetes/certs/files/k8s-certs-renew.service new file mode 100644 index 000000000..ff1100e82 --- /dev/null +++ b/builtin/core/roles/kubernetes/certs/files/k8s-certs-renew.service @@ -0,0 +1,5 @@ +[Unit] +Description=Renew K8S control plane certificates +[Service] +Type=oneshot +ExecStart=/usr/local/bin/kube-scripts/k8s-certs-renew.sh diff --git a/builtin/core/roles/kubernetes/certs/files/k8s-certs-renew.timer b/builtin/core/roles/kubernetes/certs/files/k8s-certs-renew.timer new file mode 100644 index 000000000..20f3bffda --- /dev/null +++ b/builtin/core/roles/kubernetes/certs/files/k8s-certs-renew.timer @@ -0,0 +1,7 @@ +[Unit] +Description=Timer to renew K8S control plane certificates +[Timer] +OnCalendar=Mon *-*-* 03:00:00 +Unit=k8s-certs-renew.service +[Install] +WantedBy=multi-user.target diff --git a/builtin/core/roles/kubernetes/certs/tasks/main.yaml b/builtin/core/roles/kubernetes/certs/tasks/main.yaml new file mode 100644 index 000000000..70127d862 --- /dev/null +++ b/builtin/core/roles/kubernetes/certs/tasks/main.yaml @@ -0,0 +1,19 @@ +--- +- name: Certs | Generate Kubernetes certificate renewal script + template: + src: renew_script.sh + dest: /usr/local/bin/kube-scripts/renew_script.sh + mode: 0755 + +- name: Certs | Deploy certificate renewal systemd service + copy: + src: k8s-certs-renew.service + dest: /etc/systemd/system/k8s-certs-renew.service + +- name: Certs | Deploy certificate renewal systemd timer + copy: + src: k8s-certs-renew.timer + dest: /etc/systemd/system/k8s-certs-renew.timer + +- name: Certs | Enable and start certificate renewal timer + command: systemctl daemon-reload && systemctl enable --now k8s-certs-renew.timer diff --git a/builtin/core/roles/kubernetes/certs/templates/renew_script.sh b/builtin/core/roles/kubernetes/certs/templates/renew_script.sh new file mode 100644 index 000000000..4dd9c02b3 --- /dev/null +++ b/builtin/core/roles/kubernetes/certs/templates/renew_script.sh @@ -0,0 +1,26 @@ +#!/bin/bash +{{- if .kubernetes.kube_version | semverCompare ">/dev/null >>/dev/tcp/127.0.0.1/6443; do sleep 1; done +echo "## Expiration after renewal ##" +${kubeadmCerts} check-expiration diff --git a/builtin/core/roles/kubernetes/init-kubernetes/tasks/deploy_cluster_dns.yaml b/builtin/core/roles/kubernetes/init-kubernetes/tasks/deploy_cluster_dns.yaml new file mode 100644 index 000000000..c3072c2a6 --- /dev/null +++ b/builtin/core/roles/kubernetes/init-kubernetes/tasks/deploy_cluster_dns.yaml @@ -0,0 +1,20 @@ +--- +- name: DNS | Generate CoreDNS configuration file + template: + src: dns/coredns.yaml + dest: /etc/kubernetes/coredns.yaml + +# Update the clusterIP for the service and modify the CoreDNS ConfigMap as needed +- name: DNS | Apply CoreDNS configuration and restart deployment + command: | + kubectl delete svc kube-dns -n kube-system + kubectl apply -f /etc/kubernetes/coredns.yaml && kubectl rollout restart deployment -n kube-system coredns + +- name: DNS | Generate NodeLocalDNS DaemonSet manifest + template: + src: dns/nodelocaldns.yaml + dest: /etc/kubernetes/nodelocaldns.yaml + +- name: DNS | Deploy NodeLocalDNS DaemonSet + command: | + kubectl apply -f /etc/kubernetes/nodelocaldns.yaml diff --git a/builtin/core/roles/kubernetes/init-kubernetes/tasks/init_kubernetes.yaml b/builtin/core/roles/kubernetes/init-kubernetes/tasks/init_kubernetes.yaml new file mode 100644 index 000000000..8c2e5136b --- /dev/null +++ b/builtin/core/roles/kubernetes/init-kubernetes/tasks/init_kubernetes.yaml @@ -0,0 +1,69 @@ +--- +- name: Init | Generate kubeadm initialization configuration + template: + src: >- + {{- if .kubernetes.kube_version | semverCompare ">=v1.24.0" -}} + kubeadm/kubeadm-init.v1beta3 + {{- else -}} + kubeadm/kubeadm-init.v1beta2 + {{- end -}} + dest: /etc/kubernetes/kubeadm-config.yaml + +- name: Init | Initialize Kubernetes cluster + block: + - name: Init | Pre-initialization for kube-vip + when: + - .kubernetes.kube_version | semverCompare ">=v1.29.0" + - eq .kubernetes.control_plane_endpoint.type "kube_vip" + command: | + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + - name: Init | Run kubeadm init + command: | + /usr/local/bin/kubeadm init --config=/etc/kubernetes/kubeadm-config.yaml --ignore-preflight-errors=FileExisting-crictl,ImagePull {{ if not .kubernetes.kube_proxy.enabled }}--skip-phases=addon/kube-proxy{{ end }} + - name: Init | Post-initialization for kube-vip + when: + - .kubernetes.kube_version | semverCompare ">=v1.29.0" + - eq .kubernetes.control_plane_endpoint.type "kube_vip" + command: | + sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + +# Reset local DNS for control_plane_endpoint to 127.0.0.1 and ::1. +# This ensures the control_plane_endpoint resolves locally before kube-vip is running, +# preventing failures for tasks that execute kubectl apply on the current node. +- name: Init | Reset local DNS for control_plane_endpoint + loop: "{{ .native.localDNS | toJson }}" + command: | + sed -i ':a;$!{N;ba};s@# kubekey kubernetes control_plane_endpoint BEGIN.*# kubekey kubernetes control_plane_endpoint END@@' {{ .item }} + cat >> {{ .item }} <> {{ .item }} < /etc/kubernetes/kubeadm-config.yaml + fi + - name: InitKubernetes | Fetch kubeconfig to local workspace + fetch: + src: /etc/kubernetes/admin.conf + dest: >- + {{ .work_dir }}/kubekey/kubeconfig + - name: InitKubernetes | Generate certificate key using kubeadm + command: | + /usr/local/bin/kubeadm init phase upload-certs --upload-certs --config=/etc/kubernetes/kubeadm-config.yaml 2>&1 \ + | awk '/Using certificate key:/{getline; print}' + register: kubeadm_cert_result + - name: InitKubernetes | Distribute certificate key to all cluster hosts + add_hostvars: + hosts: k8s_cluster + vars: + kubeadm_cert: >- + {{ .kubeadm_cert_result.stdout }} + - name: InitKubernetes | Generate and distribute kubeadm token + block: + - name: InitKubernetes | Generate kubeadm join token + command: /usr/local/bin/kubeadm token create + register: kubeadm_token_result + - name: InitKubernetes | Share kubeadm token with all cluster hosts + add_hostvars: + hosts: k8s_cluster + vars: + kubeadm_token: >- + {{ .kubeadm_token_result.stdout }} diff --git a/builtin/core/roles/kubernetes/init-kubernetes/templates/dns/coredns.yaml b/builtin/core/roles/kubernetes/init-kubernetes/templates/dns/coredns.yaml new file mode 100644 index 000000000..4b1131dd7 --- /dev/null +++ b/builtin/core/roles/kubernetes/init-kubernetes/templates/dns/coredns.yaml @@ -0,0 +1,115 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: coredns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + addonmanager.kubernetes.io/mode: Reconcile + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + createdby: 'kubekey' +spec: + clusterIP: {{ .dns.dns_service_ip }} + selector: + k8s-app: kube-dns + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP + - name: metrics + port: 9153 + protocol: TCP + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists +data: + Corefile: | + {{- range .dns.coredns.zone_configs }} + {{ .zones | join " " }} { + cache {{ .cache }} + {{- range .additional_configs }} + {{ . }} + {{- end }} + + {{- range .rewrite }} + rewrite {{ .rule }} { + {{ .field }} {{ .type }} {{ .value }} + {{ .options }} + } + {{- end }} + + health { + lameduck 5s + } + + {{- if .kubernetes.zones | empty | not }} + kubernetes {{ .kubernetes.zones | join " " }} in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + } + {{- end }} + + {{- range .forward }} + forward {{ .from }} {{ .to | join " " }} { + {{- if .except | empty | not }} + except {{ .except | join " " }} + {{- end }} + {{- if .force_tcp }} + force_tcp + {{- end }} + {{- if .prefer_udp }} + prefer_udp + {{- end }} + {{- if .max_fails }} + max_fails {{ .max_fails }} + {{- end }} + {{- if .expire }} + expire {{ .expire }} + {{- end }} + {{- if .tls }} + tls {{ .tls.cert_file }} {{ .tls.key_file }} {{ .tls.ca_file }} + {{- end }} + {{- if .tls_servername }} + tls_servername {{ .tls_servername }} + {{- end }} + {{- if .policy }} + policy {{ .policy }} + {{- end }} + {{- if .health_check }} + health_check {{ .health_check }} + {{- end }} + {{- if .max_concurrent }} + max_concurrent {{ .max_concurrent }} + {{- end }} + } + {{- end }} + + {{- if $.dns.dns_etc_hosts | empty | not }} + hosts /etc/coredns/hosts { + fallthrough + } + {{- end }} + } + {{- end }} + +{{- if .dns.coredns.dns_etc_hosts | empty | not }} + hosts: | + {{- range .dns.coredns.dns_etc_hosts }} + {{ . }} + {{- end }} +{{- end }} diff --git a/builtin/core/roles/kubernetes/init-kubernetes/templates/dns/nodelocaldns.yaml b/builtin/core/roles/kubernetes/init-kubernetes/templates/dns/nodelocaldns.yaml new file mode 100644 index 000000000..33815be23 --- /dev/null +++ b/builtin/core/roles/kubernetes/init-kubernetes/templates/dns/nodelocaldns.yaml @@ -0,0 +1,226 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nodelocaldns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nodelocaldns + namespace: kube-system + labels: + k8s-app: kube-dns + addonmanager.kubernetes.io/mode: Reconcile +spec: + selector: + matchLabels: + k8s-app: nodelocaldns + template: + metadata: + labels: + k8s-app: nodelocaldns + spec: + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: nodelocaldns + hostNetwork: true + dnsPolicy: Default # Don't use cluster DNS. + tolerations: + - effect: NoSchedule + operator: "Exists" + - effect: NoExecute + operator: "Exists" + - key: "CriticalAddonsOnly" + operator: "Exists" + containers: + - name: node-cache + image: {{ .dns.dns_cache_image.registry }}/{{ .dns.dns_cache_image.repository }}:{{ .dns.dns_cache_image.tag }} + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 70Mi + args: + - -localip + - {{ .dns.dns_cache_ip }} + - -conf + - /etc/coredns/Corefile + - -upstreamsvc + - coredns + - -metrics-listen-address + - 127.0.0.1:9353 + securityContext: + privileged: true + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + livenessProbe: + httpGet: + host: {{ .dns.dns_cache_ip }} + path: /health + port: 9254 + scheme: HTTP + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 10 + readinessProbe: + httpGet: + host: {{ .dns.dns_cache_ip }} + path: /health + port: 9254 + scheme: HTTP + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 10 + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + - name: xtables-lock + mountPath: /run/xtables.lock + volumes: + - name: config-volume + configMap: + name: nodelocaldns + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force + # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. + terminationGracePeriodSeconds: 0 + updateStrategy: + rollingUpdate: + maxUnavailable: 20% + type: RollingUpdate + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nodelocaldns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists +data: + Corefile: | + {{- range .dns.coredns.external_zones }} + {{ .zones | join " " }}{ + log + errors + loadbalance + cache {{ .cache }} + reload + loop + bind {{ .kubernetes.networking.dns_cache_ip }} + + {{- range .rewrite }} + rewrite {{ .rule }} { + {{ .field }} {{ .type }} {{ .value }} + {{ .options }} + } + {{- end }} + + {{- range .forward }} + forward {{ .from }} {{ .to | join " " }} { + {{- if .except | empty | not }} + except {{ .except | join " " }} + {{- end }} + {{- if .force_tcp }} + force_tcp + {{- end }} + {{ if .prefer_udp }} + prefer_udp + {{- end }} + max_fails {{ .max_fails | default 2 }} + expire {{ .expire | default "10s" }} + {{- if .tls }} + tls {{ .tls.cert_file }} {{ .tls.key_file }} {{ .tls.ca_file }} + {{- end }} + {{- if .tls_servername }} + tls_servername {{ .tls_servername }} + {{- end }} + {{- if .policy }} + policy {{ .policy }} + {{- end }} + {{- if .health_check }} + health_check {{ .health_check }} + {{- end }} + {{- if .max_concurrent }} + max_concurrent {{ .max_concurrent }} + {{- end }} + } + {{- end }} + + {{- if $.kubernetes.coredns.dns_etc_hosts | empty | not }} + hosts /etc/coredns/hosts { + fallthrough + } + {{- end }} + } + {{- end }} + + {{ .dns.dns_domain }}:53 { + errors + cache { + success 9984 30 + denial 9984 5 + } + reload + loop + bind {{ .dns.dns_cache_ip }} + forward . {{ .dns.dns_service_ip }} { + force_tcp + } + health {{ .dns.dns_cache_ip }}:9254 + } + in-addr.arpa:53 { + errors + cache 30 + reload + loop + bind {{ .dns.dns_cache_ip }} + forward . {{ .dns.dns_service_ip }} { + force_tcp + } + } + ip6.arpa:53 { + errors + cache 30 + reload + loop + bind {{ .dns.dns_cache_ip }} + forward . {{ .dns.dns_service_ip }} { + force_tcp + } + } + .:53 { + errors + cache 30 + reload + loop + bind {{ .dns.dns_cache_ip }} + forward . /etc/resolv.conf + {{- if .dns.dns_etc_hosts | empty | not }} + hosts /etc/coredns/hosts { + fallthrough + } + {{- end }} + } + +{{- if .dns.coredns.dns_etc_hosts | empty | not }} + hosts: | + {{- range .dns.coredns.dns_etc_hosts }} + {{ . }} + {{- end }} +{{- end }} diff --git a/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta2 b/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta2 new file mode 100644 index 000000000..ebb127259 --- /dev/null +++ b/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta2 @@ -0,0 +1,217 @@ +{{- $internalIPv4 := .internal_ipv4 | default "" -}} +{{- $internalIPv6 := .internal_ipv6 | default "" -}} +--- +apiVersion: kubeadm.k8s.io/v1beta2 +kind: ClusterConfiguration +etcd: +{{- if .etcd.deployment_type | eq "internal" }} + local: + imageRepository: {{ .etcd.image.registry }} + imageTag: {{ .etcd.image.tag }} + serverCertSANs: + {{- range .groups.etcd | default list }} + - {{ index $.hostvars . "internal_ipv4" }} + {{- end }} +{{- else }} + external: + endpoints: + {{- range .groups.etcd | default list }} + - https://{{ index $.hostvars . "internal_ipv4" }}:2379 + {{- end }} + caFile: /etc/kubernetes/pki/etcd/ca.crt + certFile: /etc/kubernetes/pki/etcd/client.crt + keyFile: /etc/kubernetes/pki/etcd/client.key +{{- end }} +dns: + type: CoreDNS + imageRepository: {{ .dns.dns_image.registry }}/{{ .dns.dns_image.repository }} + imageTag: {{ .dns.dns_image.tag }} +imageRepository: {{ .kubernetes.image_repository }} +kubernetesVersion: {{ .kubernetes.kube_version }} +certificatesDir: /etc/kubernetes/pki +clusterName: {{ .kubernetes.cluster_name }} +controlPlaneEndpoint: {{ .kubernetes.control_plane_endpoint.host }} +networking: + dnsDomain: {{ .dns.dns_domain }} + podSubnet: {{ .cni.pod_cidr }} + serviceSubnet: {{ .cni.service_cidr }} +apiServer: + extraArgs: +{{- if $internalIPv4 | empty | not }} + bind-address: 0.0.0.0 +{{- else if $internalIPv6 | empty | not }} + bind-address: :: +{{- end }} +{{- if .security_enhancement }} + authorization-mode: Node,RBAC + enable-admission-plugins: AlwaysPullImages,ServiceAccount,NamespaceLifecycle,NodeRestriction,LimitRanger,ResourceQuota,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,PodNodeSelector,PodSecurity + profiling: false + request-timeout: 120s + service-account-lookup: true + tls-min-version: VersionTLS12 + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 +{{- end }} +{{- if .audit }} + audit-log-format: json + audit-log-maxbackup: 2 + audit-log-maxsize: 200 + audit-policy-file: /etc/kubernetes/audit/policy.yaml + audit-webhook-config-file: /etc/kubernetes/audit/webhook.yaml +{{- end }} +{{- if .kubernetes.apiserver.extra_args | empty | not }} +{{ .kubernetes.apiserver.extra_args | toYaml | indent 4 }} +{{- end }} + certSANs: + - localhost + - 127.0.0.1 + - ::1 + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - kubernetes.default.svc.{{ .kubernetes.cluster_name }} + - kubernetes.default.svc.{{ .kubernetes.cluster_name }}.{{ .dns.dns_domain }} + - {{ index (.cni.service_cidr | ipInCIDR) 0 }} + - {{ .kubernetes.control_plane_endpoint.host }} + {{- range .groups.k8s_cluster | default list }} + - {{ index $.hostvars . "hostname" }} + - {{ index $.hostvars . "hostname" }}.{{ $.kubernetes.cluster_name }} + - {{ index $.hostvars . "hostname" }}.{{ $.kubernetes.cluster_name }}.{{ $.dns.dns_domain }} + {{- $internalIPv4 := index $.hostvars . "internal_ipv4" | default "" }} + {{- $internalIPv6 := index $.hostvars . "internal_ipv6" | default "" }} + {{- if $internalIPv4 | empty | not }} + - {{ $internalIPv4 }} + {{- end }} + {{- if $internalIPv6 | empty | not }} + - {{ $internalIPv6 }} + {{- end }} + {{- end }} + {{- range .kubernetes.apiserver.certSANs }} + - {{ . }} + {{- end }} +{{- if .audit }} + extraVolumes: + - name: k8s-audit + hostPath: /etc/kubernetes/audit + mountPath: /etc/kubernetes/audit + pathType: DirectoryOrCreate +{{- end }} +controllerManager: + extraArgs: +{{- if eq (.cni.pod_cidr | splitList "," | first | ipFamily) "IPv4" }} + node-cidr-mask-size-ipv4: "{{ .cni.ipv4_mask_size }}" +{{- end }} +{{- if eq (.cni.pod_cidr | splitList "," | last | ipFamily) "IPv6" }} + node-cidr-mask-size-ipv6: "{{ .cni.ipv6_mask_size }}" +{{- end }} +{{- if .security_enhancement }} + {{- if $internalIPv4 | empty | not }} + bind-address: 127.0.0.1 + {{- else if $internalIPv6 | empty | not }} + bind-address: ::1 + {{- end }} + profiling: false + terminated-pod-gc-threshold: 50 + use-service-account-credentials: true +{{- else }} + {{- if $internalIPv4 | empty | not }} + bind-address: 0.0.0.0 + {{- else if $internalIPv6 | empty | not }} + bind-address: :: + {{- end }} +{{- end }} +{{- if .kubernetes.controller_manager.extra_args }} +{{ .kubernetes.controller_manager.extra_args | toYaml | indent 4 }} +{{- end }} + extraVolumes: + - name: host-time + hostPath: /etc/localtime + mountPath: /etc/localtime + readOnly: true +scheduler: + extraArgs: +{{- if .security_enhancement }} + {{- if $internalIPv4 | empty | not }} + bind-address: 127.0.0.1 + {{- else if $internalIPv6 | empty | not }} + bind-address: ::1 + {{- end }} + profiling: false +{{- else }} + {{- if $internalIPv4 | empty | not }} + bind-address: 0.0.0.0 + {{- else if $internalIPv6 | empty | not }} + bind-address: :: + {{- end }} +{{- end }} +{{- if .kubernetes.scheduler.extra_args }} +{{ .kubernetes.scheduler.extra_args | toYaml | indent 4 }} +{{- end }} +--- +apiVersion: kubeadm.k8s.io/v1beta2 +kind: InitConfiguration +localAPIEndpoint: +{{- if $internalIPv4 | empty | not }} + advertiseAddress: {{ $internalIPv4 }} +{{- else if $internalIPv6 | empty | not }} + advertiseAddress: {{ $internalIPv6 }} +{{- end }} + bindPort: {{ .kubernetes.apiserver.port }} +nodeRegistration: + criSocket: {{ .cri.cri_socket }} + kubeletExtraArgs: + cgroup-driver: {{ .cri.cgroup_driver }} + pod-infra-container-image: "{{ .cri.sandbox_image.registry }}/{{ .cri.sandbox_image.repository }}:{{ .cri.sandbox_image.tag }}" + +--- +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +kind: KubeProxyConfiguration +clusterCIDR: {{ .cni.pod_cidr }} +mode: {{ .kubernetes.kube_proxy.mode }} +{{- if .kubernetes.kube_proxy.config | empty | not }} +{{ .kubernetes.kube_proxy.config | toYaml }} +{{- end }} +--- +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +clusterDomain: {{ .dns.dns_domain }} +clusterDNS: + - {{ .dns.dns_cache_ip }} +maxPods: {{ .cni.max_pods }} +podPidsLimit: {{ .kubernetes.kubelet.pod_pids_limit }} +rotateCertificates: true +kubeReserved: + cpu: 200m + memory: 250Mi +systemReserved: + cpu: 200m + memory: 250Mi +evictionHard: + memory.available: 5% + pid.available: 10% +evictionSoft: + memory.available: 10% +evictionSoftGracePeriod: + memory.available: 2m +evictionMaxPodGracePeriod: 120 +evictionPressureTransitionPeriod: 30s +{{- if .security_enhancement }} +readOnlyPort: 0 +protectKernelDefaults: true +eventRecordQPS: 1 +streamingConnectionIdleTimeout: 5m +makeIPTablesUtilChains: true +tlsCipherSuites: + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 +{{- end }} +{{- if .kubernetes.kubelet.feature_gates | empty | not }} +featureGates: +{{ .kubernetes.kubelet.feature_gates | toYaml | indent 2 }} +{{- end }} +cgroupDriver: {{ .cri.cgroup_driver }} +containerLogMaxSize: {{ .kubernetes.kubelet.container_log_max_size }} +containerLogMaxFiles: {{ .kubernetes.kubelet.container_log_max_files }} +{{- if .kubernetes.kubelet.extra_args | empty | not }} +{{ .kubernetes.kubelet.extra_args | toYaml }} +{{- end }} diff --git a/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta3 b/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta3 new file mode 100644 index 000000000..b594d9d37 --- /dev/null +++ b/builtin/core/roles/kubernetes/init-kubernetes/templates/kubeadm/kubeadm-init.v1beta3 @@ -0,0 +1,216 @@ +{{- $internalIPv4 := .internal_ipv4 | default "" -}} +{{- $internalIPv6 := .internal_ipv6 | default "" -}} +--- +apiVersion: kubeadm.k8s.io/v1beta3 +kind: ClusterConfiguration +etcd: +{{- if .etcd.deployment_type | eq "internal" }} + local: + imageRepository: {{ .etcd.image.registry }} + imageTag: {{ .etcd.image.tag }} + serverCertSANs: + {{- range .groups.etcd | default list }} + - {{ index $.hostvars . "internal_ipv4" }} + {{- end }} +{{- else }} + external: + endpoints: + {{- range .groups.etcd | default list }} + - https://{{ index $.hostvars . "internal_ipv4" }}:2379 + {{- end }} + caFile: /etc/kubernetes/pki/etcd/ca.crt + certFile: /etc/kubernetes/pki/etcd/client.crt + keyFile: /etc/kubernetes/pki/etcd/client.key +{{- end }} +dns: + imageRepository: {{ .dns.dns_image.registry }}/{{ .dns.dns_image.repository }} + imageTag: {{ .dns.dns_image.tag }} +imageRepository: {{ .kubernetes.image_repository }} +kubernetesVersion: {{ .kubernetes.kube_version }} +certificatesDir: /etc/kubernetes/pki +clusterName: {{ .kubernetes.cluster_name }} +controlPlaneEndpoint: {{ .kubernetes.control_plane_endpoint.host }} +networking: + dnsDomain: {{ .dns.dns_domain }} + podSubnet: {{ .cni.pod_cidr }} + serviceSubnet: {{ .cni.service_cidr }} +apiServer: + extraArgs: +{{- if $internalIPv4 | empty | not }} + bind-address: 0.0.0.0 +{{- else if $internalIPv6 | empty | not }} + bind-address: :: +{{- end }} +{{- if .security_enhancement }} + authorization-mode: Node,RBAC + enable-admission-plugins: AlwaysPullImages,ServiceAccount,NamespaceLifecycle,NodeRestriction,LimitRanger,ResourceQuota,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,PodNodeSelector,PodSecurity + profiling: false + request-timeout: 120s + service-account-lookup: true + tls-min-version: VersionTLS12 + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 +{{- end }} +{{- if .audit }} + audit-log-format: json + audit-log-maxbackup: 2 + audit-log-maxsize: 200 + audit-policy-file: /etc/kubernetes/audit/policy.yaml + audit-webhook-config-file: /etc/kubernetes/audit/webhook.yaml +{{- end }} +{{- if .kubernetes.apiserver.extra_args | empty | not }} +{{ .kubernetes.apiserver.extra_args | toYaml | indent 4 }} +{{- end }} + certSANs: + - localhost + - 127.0.0.1 + - ::1 + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - kubernetes.default.svc.{{ .kubernetes.cluster_name }} + - kubernetes.default.svc.{{ .kubernetes.cluster_name }}.{{ .dns.dns_domain }} + - {{ index (.cni.service_cidr | ipInCIDR) 0 }} + - {{ .kubernetes.control_plane_endpoint.host }} + {{- range .groups.k8s_cluster | default list }} + - {{ index $.hostvars . "hostname" }} + - {{ index $.hostvars . "hostname" }}.{{ $.kubernetes.cluster_name }} + - {{ index $.hostvars . "hostname" }}.{{ $.kubernetes.cluster_name }}.{{ $.dns.dns_domain }} + {{- $internalIPv4 := index $.hostvars . "internal_ipv4" | default "" }} + {{- $internalIPv6 := index $.hostvars . "internal_ipv6" | default "" }} + {{- if $internalIPv4 | empty | not }} + - {{ $internalIPv4 }} + {{- end }} + {{- if $internalIPv6 | empty | not }} + - {{ $internalIPv6 }} + {{- end }} + {{- end }} + {{- range .kubernetes.apiserver.certSANs }} + - {{ . }} + {{- end }} +{{- if .audit }} + extraVolumes: + - name: k8s-audit + hostPath: /etc/kubernetes/audit + mountPath: /etc/kubernetes/audit + pathType: DirectoryOrCreate +{{- end }} +controllerManager: + extraArgs: +{{- if eq (.cni.pod_cidr | splitList "," | first | ipFamily) "IPv4" }} + node-cidr-mask-size-ipv4: "{{ .cni.ipv4_mask_size }}" +{{- end }} +{{- if eq (.cni.pod_cidr | splitList "," | last | ipFamily) "IPv6" }} + node-cidr-mask-size-ipv6: "{{ .cni.ipv6_mask_size }}" +{{- end }} +{{- if .security_enhancement }} + {{- if $internalIPv4 | empty | not }} + bind-address: 127.0.0.1 + {{- else if $internalIPv6 | empty | not }} + bind-address: ::1 + {{- end }} + profiling: false + terminated-pod-gc-threshold: 50 + use-service-account-credentials: true +{{- else }} + {{- if $internalIPv4 | empty | not }} + bind-address: 0.0.0.0 + {{- else if $internalIPv6 | empty | not }} + bind-address: :: + {{- end }} +{{- end }} +{{- if .kubernetes.controller_manager.extra_args }} +{{ .kubernetes.controller_manager.extra_args | toYaml | indent 4 }} +{{- end }} + extraVolumes: + - name: host-time + hostPath: /etc/localtime + mountPath: /etc/localtime + readOnly: true +scheduler: + extraArgs: +{{- if .security_enhancement }} + {{- if $internalIPv4 | empty | not }} + bind-address: 127.0.0.1 + {{- else if $internalIPv6 | empty | not }} + bind-address: ::1 + {{- end }} + profiling: false +{{- else }} + {{- if $internalIPv4 | empty | not }} + bind-address: 0.0.0.0 + {{- else if $internalIPv6 | empty | not }} + bind-address: :: + {{- end }} +{{- end }} +{{- if .kubernetes.scheduler.extra_args }} +{{ .kubernetes.scheduler.extra_args | toYaml | indent 4 }} +{{- end }} +--- +apiVersion: kubeadm.k8s.io/v1beta3 +kind: InitConfiguration +localAPIEndpoint: +{{- if $internalIPv4 | empty | not }} + advertiseAddress: {{ $internalIPv4 }} +{{- else if $internalIPv6 | empty | not }} + advertiseAddress: {{ $internalIPv6 }} +{{- end }} + bindPort: {{ .kubernetes.apiserver.port }} +nodeRegistration: + criSocket: {{ .cri.cri_socket }} + kubeletExtraArgs: + cgroup-driver: {{ .cri.cgroup_driver }} + pod-infra-container-image: "{{ .cri.sandbox_image.registry }}/{{ .cri.sandbox_image.repository }}:{{ .cri.sandbox_image.tag }}" + +--- +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +kind: KubeProxyConfiguration +clusterCIDR: {{ .cni.pod_cidr }} +mode: {{ .kubernetes.kube_proxy.mode }} +{{- if .kubernetes.kube_proxy.config | empty | not }} +{{ .kubernetes.kube_proxy.config | toYaml }} +{{- end }} +--- +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +clusterDomain: {{ .dns.dns_domain }} +clusterDNS: + - {{ .dns.dns_cache_ip }} +maxPods: {{ .cni.max_pods }} +podPidsLimit: {{ .kubernetes.kubelet.pod_pids_limit }} +rotateCertificates: true +kubeReserved: + cpu: 200m + memory: 250Mi +systemReserved: + cpu: 200m + memory: 250Mi +evictionHard: + memory.available: 5% + pid.available: 10% +evictionSoft: + memory.available: 10% +evictionSoftGracePeriod: + memory.available: 2m +evictionMaxPodGracePeriod: 120 +evictionPressureTransitionPeriod: 30s +{{- if .security_enhancement }} +readOnlyPort: 0 +protectKernelDefaults: true +eventRecordQPS: 1 +streamingConnectionIdleTimeout: 5m +makeIPTablesUtilChains: true +tlsCipherSuites: + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 +{{- end }} +{{- if .kubernetes.kubelet.feature_gates | empty | not }} +featureGates: +{{ .kubernetes.kubelet.feature_gates | toYaml | indent 2 }} +{{- end }} +cgroupDriver: {{ .cri.cgroup_driver }} +containerLogMaxSize: {{ .kubernetes.kubelet.container_log_max_size }} +containerLogMaxFiles: {{ .kubernetes.kubelet.container_log_max_files }} +{{- if .kubernetes.kubelet.extra_args | empty | not }} +{{ .kubernetes.kubelet.extra_args | toYaml }} +{{- end }} diff --git a/builtin/core/roles/kubernetes/join-kubernetes/tasks/main.yaml b/builtin/core/roles/kubernetes/join-kubernetes/tasks/main.yaml new file mode 100644 index 000000000..a3a7fcb8d --- /dev/null +++ b/builtin/core/roles/kubernetes/join-kubernetes/tasks/main.yaml @@ -0,0 +1,68 @@ +--- +- name: Join | Generate kubeadm join configuration file + template: + src: >- + {{- if .kubernetes.kube_version | semverCompare ">=v1.24.0" -}} + kubeadm/kubeadm-join.v1beta3 + {{- else -}} + kubeadm/kubeadm-join.v1beta2 + {{- end -}} + dest: /etc/kubernetes/kubeadm-config.yaml + +- name: Join | Execute kubeadm join to add node to the Kubernetes cluster + command: | + /usr/local/bin/kubeadm join --config=/etc/kubernetes/kubeadm-config.yaml --ignore-preflight-errors=FileExisting-crictl,ImagePull + +- name: Join | Synchronize kubeconfig to remote node + copy: + src: >- + {{ .work_dir }}/kubekey/kubeconfig + dest: /root/.kube/config + +- name: Join | Configure node as worker + when: .groups.kube_worker | default list | has .inventory_hostname + block: + - name: Join | Remove master and control-plane taints from node + ignore_errors: true + command: | + /usr/local/bin/kubectl taint nodes {{ .hostname }} node-role.kubernetes.io/master=:NoSchedule- + /usr/local/bin/kubectl taint nodes {{ .hostname }} node-role.kubernetes.io/control-plane=:NoSchedule- + - name: Join | Add worker label to node + command: | + /usr/local/bin/kubectl label --overwrite node {{ .hostname }} node-role.kubernetes.io/worker= + +- name: Join | Add custom annotations to node + when: .annotations | empty | not + command: | + kubectl annotate {{ .hostname }} {{- range $k,$v := .annotations }}{{ printf "%s=%s" $k $v}} {{- end }} + +# Reset local DNS for control_plane_endpoint to 127.0.0.1 and ::1. +# This ensures the control_plane_endpoint resolves locally before kube-vip is running, +# preventing failures for tasks that execute kubectl apply on the current node. +- name: Join | Reset local DNS for control_plane_endpoint + block: + - name: Join | Reset local DNS on control plane nodes + when: + - .groups.kube_control_plane | default list | has .inventory_hostname + loop: "{{ .native.localDNS | toJson }}" + command: | + sed -i ':a;$!{N;ba};s@# kubekey kubernetes control_plane_endpoint BEGIN.*# kubekey kubernetes control_plane_endpoint END@@' {{ .item }} + cat >> {{ .item }} <> {{ .item }} <- + {{- $interface := "" }} + {{- range .interface.stdout | default list -}} + {{- if .cidr | ipInCIDR | has $.kubernetes.control_plane_endpoint.kube_vip.address -}} + {{- $interface = .interface -}} + {{- end -}} + {{- end -}} + {{ $interface }} + - name: Kubevip | Ensure matching network interface exists + assert: + that: .kube_vip_interface | empty + fail_msg: "Kubevip: Unable to find a network interface matching the kube-vip address." + +- name: Kubevip | Generate kube-vip manifest file + template: + src: >- + kubevip/kubevip.{{ .kubernetes.control_plane_endpoint.kube_vip.mode }} + dest: /etc/kubernetes/manifests/kube-vip.yaml diff --git a/builtin/core/roles/kubernetes/pre-kubernetes/tasks/high-availability/main.yaml b/builtin/core/roles/kubernetes/pre-kubernetes/tasks/high-availability/main.yaml new file mode 100644 index 000000000..e2da00519 --- /dev/null +++ b/builtin/core/roles/kubernetes/pre-kubernetes/tasks/high-availability/main.yaml @@ -0,0 +1,14 @@ +# HighAvailability: Install kube-vip as a static pod. See: https://kube-vip.io/docs/installation/static/ +- name: HighAvailability | Include kube-vip static pod tasks + include_tasks: high-availability/kube_vip.yaml + when: + - .kubernetes.control_plane_endpoint.type | eq "kube_vip" + - .groups.kube_control_plane | default list | has .inventory_hostname + +# HighAvailability: Deploy HAProxy only on worker nodes. Control plane nodes use the local static pod for kube-apiserver. +- name: HighAvailability | Include HAProxy deployment tasks for worker nodes + include_tasks: high-availability/haproxy.yaml + when: + - .kubernetes.control_plane_endpoint.type | eq "haproxy" + - .groups.kube_worker | default list | has .inventory_hostname + - .groups.kube_control_plane | default list | has .inventory_hostname | not \ No newline at end of file diff --git a/builtin/core/roles/kubernetes/pre-kubernetes/tasks/install_binaries.yaml b/builtin/core/roles/kubernetes/pre-kubernetes/tasks/install_binaries.yaml new file mode 100644 index 000000000..0b1c8e623 --- /dev/null +++ b/builtin/core/roles/kubernetes/pre-kubernetes/tasks/install_binaries.yaml @@ -0,0 +1,84 @@ +--- +- name: Binary | Verify if Helm is already installed + ignore_errors: true + command: helm version --template "{{ .Version }}" + register: helm_install_version + +- name: Binary | Install Helm if not present or version mismatch + when: or (.helm_install_version.error | empty | not) (.helm_install_version.stdout | ne .kubernetes.helm_version) + block: + - name: Binary | Copy Helm archive to remote host + copy: + src: >- + {{ .binary_dir }}/helm/{{ .kubernetes.helm_version }}/{{ .binary_type }}/helm-{{ .kubernetes.helm_version }}-linux-{{ .binary_type }}.tar.gz + dest: >- + {{ .tmp_dir }}/helm-{{ .kubernetes.helm_version }}-linux-{{ .binary_type }}.tar.gz + - name: Binary | Extract and install Helm binary + command: | + tar --strip-components=1 -zxvf {{ .tmp_dir }}/helm-{{ .kubernetes.helm_version }}-linux-{{ .binary_type }}.tar.gz -C /usr/local/bin linux-{{ .binary_type }}/helm + +- name: Binary | Check if kubeadm is installed + ignore_errors: true + command: kubeadm version -o short + register: kubeadm_install_version + +- name: Binary | Install kubeadm if not present or version mismatch + when: or (.kubeadm_install_version.error | empty | not) (.kubeadm_install_version.stdout | ne .kubernetes.kube_version) + copy: + src: >- + {{ .binary_dir }}/kube/{{ .kubernetes.kube_version }}/{{ .binary_type }}/kubeadm + dest: /usr/local/bin/kubeadm + mode: 0755 + +- name: Binary | Check if kubectl is installed + ignore_errors: true + command: kubectl version --short + register: kubectl_install_version + register_type: yaml + +- name: Binary | Install kubectl if not present or version mismatch + when: | + or (.kubectl_install_version.error | empty | not) ((get .kubectl_install_version.stdout "Server Version") | ne .kubernetes.kube_version) + copy: + src: >- + {{ .binary_dir }}/kube/{{ .kubernetes.kube_version }}/{{ .binary_type }}/kubectl + dest: /usr/local/bin/kubectl + mode: 0755 + +- name: Binary | Check if kubelet is installed + ignore_errors: true + command: kubelet --version + register: kubelet_install_version + +- name: Binary | Install kubelet if not present or version mismatch + when: or (.kubelet_install_version.error | empty | not) (.kubelet_install_version.stdout | ne (printf "Kubernetes %s" .kubernetes.kube_version)) + block: + - name: Binary | Copy kubelet binary to remote host + copy: + src: >- + {{ .binary_dir }}/kube/{{ .kubernetes.kube_version }}/{{ .binary_type }}/kubelet + dest: /usr/local/bin/kubelet + mode: 0755 + - name: Binary | Deploy kubelet environment configuration + template: + src: kubelet/kubelet.env + dest: /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + - name: Binary | Copy kubelet systemd service file + copy: + src: kubelet.service + dest: /etc/systemd/system/kubelet.service + - name: Binary | Reload systemd and enable kubelet service + command: systemctl daemon-reload && systemctl enable kubelet.service + +- name: Binary | Install CNI plugins if version specified + when: .cni.cni_plugins_version | empty | not + block: + - name: Binary | Copy CNI plugins archive to remote host + copy: + src: >- + {{ .binary_dir }}/cni/plugins/{{ .cni.cni_plugins_version }}/{{ .binary_type }}/cni-plugins-linux-{{ .binary_type }}-{{ .cni.cni_plugins_version }}.tgz + dest: >- + {{ .tmp_dir }}/cni-plugins-linux-{{ .binary_type }}-{{ .cni.cni_plugins_version }}.tgz + - name: Binary | Extract and install CNI plugins + command: | + tar -zxvf {{ .tmp_dir }}/cni-plugins-linux-{{ .binary_type }}-{{ .cni.cni_plugins_version }}.tgz -C /opt/cni/bin/ \ No newline at end of file diff --git a/builtin/core/roles/kubernetes/pre-kubernetes/tasks/main.yaml b/builtin/core/roles/kubernetes/pre-kubernetes/tasks/main.yaml new file mode 100644 index 000000000..f4a28bb34 --- /dev/null +++ b/builtin/core/roles/kubernetes/pre-kubernetes/tasks/main.yaml @@ -0,0 +1,84 @@ +- include_tasks: install_binaries.yaml + +- include_tasks: high-availability/main.yaml + +- name: PreKubernetes | Ensure the 'kube' system user exists + command: | + id kube &>/dev/null || useradd -M -c 'Kubernetes user' -s /sbin/nologin -r kube + +- name: PreKubernetes | Create and set ownership for required Kubernetes directories + command: | + if [ ! -d "{{ .item.path }}" ]; then + mkdir -p {{ .item.path }} && chown kube -R {{ .item.chown }} + fi + loop: + - {path: "/usr/local/bin", chown: "/usr/local/bin"} + - {path: "/etc/kubernetes", chown: "/etc/kubernetes"} + - {path: "/etc/kubernetes/pki", chown: "/etc/kubernetes/pki"} + - {path: "/etc/kubernetes/manifests", chown: "/etc/kubernetes/manifests"} + - {path: "/usr/local/bin/kube-scripts", chown: "/usr/local/bin/kube-scripts"} + - {path: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec", chown: "/usr/libexec/kubernetes"} + - {path: "/etc/cni/net.d", chown: "/etc/cni"} + - {path: "/opt/cni/bin", chown: "/opt/cni"} + - {path: "/var/lib/calico", chown: "/var/lib/calico"} + +- name: PreKubernetes | Synchronize audit policy file to remote node + copy: + src: audit + dest: /etc/kubernetes/audit/ + when: .audit + +- name: PreKubernetes | Synchronize cluster CA files to control plane nodes + when: + - .kubernetes.certs.ca_cert | empty | not + - .kubernetes.certs.ca_key | empty | not + - .groups.kube_control_plane | has .inventory_hostname + block: + - name: PreKubernetes | Copy CA certificate to control plane node + copy: + src: >- + {{ .kubernetes.certs.ca_cert }} + dest: /etc/kubernetes/pki/ca.crt + - name: PreKubernetes | Copy CA private key to control plane node + copy: + src: >- + {{ .kubernetes.certs.ca_key }} + dest: /etc/kubernetes/pki/ca.key + +- name: PreKubernetes | Ensure external etcd certificates are present on control plane nodes + when: + - .etcd.deployment_type | eq "external" + - .groups.kube_control_plane | default list | has .inventory_hostname + block: + - name: PreKubernetes | Copy etcd CA certificate to control plane node + copy: + src: >- + {{ .etcd.ca_file }} + dest: /etc/kubernetes/pki/etcd/ca.crt + - name: PreKubernetes | Copy etcd client certificate to control plane node + copy: + src: >- + {{ .etcd.cert_file }} + dest: /etc/kubernetes/pki/etcd/client.crt + - name: PreKubernetes | Copy etcd client key to control plane node + copy: + src: >- + {{ .etcd.key_file }} + dest: /etc/kubernetes/pki/etcd/client.key + +- name: PreKubernetes | Synchronize front-proxy CA files to control plane nodes + when: + - .kubernetes.certs.front_proxy_cert | empty | not + - .kubernetes.certs.front_proxy_key | empty | not + - .groups.kube_control_plane | has .inventory_hostname + block: + - name: PreKubernetes | Copy front-proxy CA certificate to control plane node + copy: + src: >- + {{ .kubernetes.certs.front_proxy_cert }} + dest: /etc/kubernetes/pki/front-proxy-ca.crt + - name: PreKubernetes | Copy front-proxy CA private key to control plane node + copy: + src: >- + {{ .kubernetes.certs.front_proxy_key }} + dest: /etc/kubernetes/pki/front-proxy-ca.key \ No newline at end of file diff --git a/builtin/core/roles/kubernetes/pre-kubernetes/templates/haproxy/haproxy.cfg b/builtin/core/roles/kubernetes/pre-kubernetes/templates/haproxy/haproxy.cfg new file mode 100644 index 000000000..22da18c25 --- /dev/null +++ b/builtin/core/roles/kubernetes/pre-kubernetes/templates/haproxy/haproxy.cfg @@ -0,0 +1,47 @@ +global + maxconn 4000 + log 127.0.0.1 local0 + +defaults + mode http + log global + option httplog + option dontlognull + option http-server-close + option redispatch + retries 5 + timeout http-request 5m + timeout queue 5m + timeout connect 30s + timeout client 30s + timeout server 15m + timeout http-keep-alive 30s + timeout check 30s + maxconn 4000 + +frontend healthz + bind *:{{ .kubernetes.control_plane_endpoint.haproxy.health_port }} + mode http + monitor-uri /healthz + +frontend kube_api_frontend + bind {{ .kubernetes.control_plane_endpoint.haproxy.address }}:{{ .kubernetes.control_plane_endpoint.port }} + mode tcp + option tcplog + default_backend kube_api_backend + +backend kube_api_backend + mode tcp + balance leastconn + default-server inter 15s downinter 15s rise 2 fall 2 slowstart 60s maxconn 1000 maxqueue 256 weight 100 + option httpchk GET /healthz + http-check expect status 200 +{{- range .groups.kube_control_plane | default list }} + {{- $internalIPv4 := index $.hostvars . "internal_ipv4" | default "" }} + {{- $internalIPv6 := index $.hostvars . "internal_ipv6" | default "" }} + {{- if $internalIPv4 | empty | not }} + server {{ index $.hostvars . "hostname" }} {{ $internalIPv4 }}:{{ $.kubernetes.apiserver.port }} check check-ssl verify none + {{- else if $internalIPv6 | empty | not }} + server {{ index $.hostvars . "hostname" }} {{ $internalIPv6 }}:{{ $.kubernetes.apiserver.port }} check check-ssl verify none + {{- end }} +{{- end }} diff --git a/builtin/core/roles/kubernetes/pre-kubernetes/templates/haproxy/haproxy.yaml b/builtin/core/roles/kubernetes/pre-kubernetes/templates/haproxy/haproxy.yaml new file mode 100644 index 000000000..85d463c96 --- /dev/null +++ b/builtin/core/roles/kubernetes/pre-kubernetes/templates/haproxy/haproxy.yaml @@ -0,0 +1,41 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: haproxy + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile + k8s-app: kube-haproxy + annotations: + cfg-checksum: "{{ .haproxy_cfg_md5.stdout }}" +spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + containers: + - name: haproxy + image: {{ .kubernetes.control_plane_endpoint.haproxy.image.registry }}/{{ .kubernetes.control_plane_endpoint.haproxy.image.repository }}:{{ .kubernetes.control_plane_endpoint.haproxy.image.tag }} + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 25m + memory: 32M + livenessProbe: + httpGet: + path: /healthz + port: {{ .kubernetes.control_plane_endpoint.haproxy.health_port }} + readinessProbe: + httpGet: + path: /healthz + port: {{ .kubernetes.control_plane_endpoint.haproxy.health_port }} + volumeMounts: + - mountPath: /usr/local/etc/haproxy/ + name: etc-haproxy + readOnly: true + volumes: + - name: etc-haproxy + hostPath: + path: /etc/kubekey/haproxy diff --git a/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubelet/kubelet.env b/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubelet/kubelet.env new file mode 100644 index 000000000..4ec1e05d7 --- /dev/null +++ b/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubelet/kubelet.env @@ -0,0 +1,18 @@ +# Note: This dropin only works with kubeadm and kubelet v1.11+ +[Service] +Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" +Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" +# This is a file that "kubeadm init" and "kubeadm join" generate at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically +EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env +# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use +# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file. +EnvironmentFile=-/etc/default/kubelet +{{- $internalIPv4 := .internal_ipv4 | default "" }} +{{- $internalIPv6 := .internal_ipv6 | default "" }} +{{- if $internalIPv4 | empty | not }} +Environment="KUBELET_EXTRA_ARGS=--node-ip={{ $internalIPv4 }} --hostname-override={{ .hostname }} {{ range $k,$v := .kubernetes.kubelet.extra_args }}--{{ $k }} {{ $v }} {{ end }}" +{{- else if $internalIPv6 | empty | not }} +Environment="KUBELET_EXTRA_ARGS=--node-ip={{ $internalIPv6 }} --hostname-override={{ .hostname }} {{ range $k,$v := .kubernetes.kubelet.extra_args }}--{{ $k }} {{ $v }} {{ end }}" +{{- end }} +ExecStart= +ExecStart=/usr/local/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS diff --git a/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubevip/kubevip.ARP b/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubevip/kubevip.ARP new file mode 100644 index 000000000..5b7fb6efa --- /dev/null +++ b/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubevip/kubevip.ARP @@ -0,0 +1,64 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: kube-vip + namespace: kube-system +spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_interface + value: {{ .kube_vip_interface }} + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "true" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: lb_enable + value: "true" + - name: lb_port + value: "6443" + - name: address + value: {{ .kubernetes.control_plane_endpoint.kube_vip.address }} + image: {{ .kubernetes.control_plane_endpoint.kube_vip.image.registry }}/{{ .kubernetes.control_plane_endpoint.kube_vip.image.repository }}:{{ .kubernetes.control_plane_endpoint.kube_vip.image.tag }} + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + - SYS_TIME + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: kubeconfig +status: {} diff --git a/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubevip/kubevip.BGP b/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubevip/kubevip.BGP new file mode 100644 index 000000000..b4be82bae --- /dev/null +++ b/builtin/core/roles/kubernetes/pre-kubernetes/templates/kubevip/kubevip.BGP @@ -0,0 +1,81 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: kube-vip + namespace: kube-system +spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "false" + - name: port + value: "6443" + - name: vip_interface + value: {{ .kube_vip_interface }} + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "true" + - name: bgp_enable + value: "true" + - name: bgp_routerid + value: | + {{- $ips := list }} + {{- range .groups.kube_control_plane | default list }} + {{- $ips = append $ips (index $.hostvars . "internal_ipv4") }} + {{- end }} + {{ $ips | join "," }} + - name: bgp_as + value: "65000" + - name: bgp_peeraddress + - name: bgp_peerpass + - name: bgp_peeras + value: "65000" + - name: bgp_peers + value: | + {{- $ips := list }} + {{- range .groups.kube_control_plane | default list }} + {{- $ips = append $ips (printf "%s:65000::false" (index $.hostvars . "internal_ipv4")) }} + {{- end }} + {{ $ips | join "," }} + - name: lb_enable + value: "true" + - name: lb_port + value: "6443" + - name: lb_fwdmethod + value: local + - name: address + value: {{ .kubernetes.control_plane_endpoint.kube_vip.address }} + - name: prometheus_server + value: :2112 + image: {{ .kubernetes.control_plane_endpoint.kube_vip.image.registry }}/{{ .kubernetes.control_plane_endpoint.kube_vip.image.repository }}:{{ .kubernetes.control_plane_endpoint.kube_vip.image.tag }} + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: kubeconfig +status: {} diff --git a/builtin/core/roles/native/dns/tasks/main.yaml b/builtin/core/roles/native/dns/tasks/main.yaml new file mode 100644 index 000000000..926594e2b --- /dev/null +++ b/builtin/core/roles/native/dns/tasks/main.yaml @@ -0,0 +1,149 @@ +- name: DNS | Ensure local DNS entries are up-to-date + loop: "{{ .native.localDNS | toJson }}" + command: | + # Clean up any previous Kubekey-managed DNS blocks + sed -i ':a;$!{N;ba};s@# kubekey hosts BEGIN.*# kubekey hosts END@@' {{ .item }} + sed -i '/^$/N;/\n$/N;//D' {{ .item }} + # Write the latest Kubekey DNS configuration + cat >> {{ .item }} <> {{ .item }} <> {{ .item }} < /dev/null +then + setenforce 0 + getenforce +fi + +# ------------------------ 2. Network Settings (Sysctl) ------------------------ + +echo 'net.core.netdev_max_backlog = 65535' >> /etc/sysctl.conf +echo 'net.core.rmem_max = 33554432' >> /etc/sysctl.conf +echo 'net.core.wmem_max = 33554432' >> /etc/sysctl.conf +echo 'net.core.somaxconn = 32768' >> /etc/sysctl.conf +echo 'net.bridge.bridge-nf-call-arptables = 1' >> /etc/sysctl.conf +echo 'vm.max_map_count = 262144' >> /etc/sysctl.conf +echo 'vm.swappiness = 0' >> /etc/sysctl.conf +echo 'vm.overcommit_memory = 1' >> /etc/sysctl.conf +echo 'fs.inotify.max_user_instances = 524288' >> /etc/sysctl.conf +echo 'fs.inotify.max_user_watches = 10240001' >> /etc/sysctl.conf +echo 'fs.pipe-max-size = 4194304' >> /etc/sysctl.conf +echo 'fs.aio-max-nr = 262144' >> /etc/sysctl.conf +echo 'kernel.pid_max = 65535' >> /etc/sysctl.conf +echo 'kernel.watchdog_thresh = 5' >> /etc/sysctl.conf +echo 'kernel.hung_task_timeout_secs = 5' >> /etc/sysctl.conf +{{- if .internal_ipv4 | empty | not }} +# add for ipv4 +echo 'net.ipv4.ip_forward = 1' >> /etc/sysctl.conf +echo 'net.bridge.bridge-nf-call-ip6tables = 1' >> /etc/sysctl.conf +echo 'net.ipv4.ip_local_reserved_ports = 30000-32767' >> /etc/sysctl.conf +echo 'net.ipv4.tcp_max_syn_backlog = 1048576' >> /etc/sysctl.conf +echo 'net.ipv4.neigh.default.gc_thresh1 = 512' >> /etc/sysctl.conf +echo 'net.ipv4.neigh.default.gc_thresh2 = 2048' >> /etc/sysctl.conf +echo 'net.ipv4.neigh.default.gc_thresh3 = 4096' >> /etc/sysctl.conf +echo 'net.ipv4.tcp_retries2 = 15' >> /etc/sysctl.conf +echo 'net.ipv4.tcp_max_tw_buckets = 1048576' >> /etc/sysctl.conf +echo 'net.ipv4.tcp_max_orphans = 65535' >> /etc/sysctl.conf +echo 'net.ipv4.udp_rmem_min = 131072' >> /etc/sysctl.conf +echo 'net.ipv4.udp_wmem_min = 131072' >> /etc/sysctl.conf +echo 'net.ipv4.conf.all.rp_filter = 1' >> /etc/sysctl.conf +echo 'net.ipv4.conf.default.rp_filter = 1' >> /etc/sysctl.conf +echo 'net.ipv4.conf.all.arp_accept = 1' >> /etc/sysctl.conf +echo 'net.ipv4.conf.default.arp_accept = 1' >> /etc/sysctl.conf +echo 'net.ipv4.conf.all.arp_ignore = 1' >> /etc/sysctl.conf +echo 'net.ipv4.conf.default.arp_ignore = 1' >> /etc/sysctl.conf +{{- end }} +{{- if .internal_ipv6 | empty | not }} +# add for ipv6 +echo 'net.bridge.bridge-nf-call-iptables = 1' >> /etc/sysctl.conf +echo 'net.ipv6.conf.all.disable_ipv6 = 0' >> /etc/sysctl.conf +echo 'net.ipv6.conf.default.disable_ipv6 = 0' >> /etc/sysctl.conf +echo 'net.ipv6.conf.lo.disable_ipv6 = 0' >> /etc/sysctl.conf +echo 'net.ipv6.conf.all.forwarding=1' >> /etc/sysctl.conf +echo 'net.ipv6.conf.default.accept_dad=0' >> /etc/sysctl.conf +echo 'net.ipv6.route.max_size=65536' >> /etc/sysctl.conf +echo 'net.ipv6.neigh.default.retrans_time_ms=1000' >> /etc/sysctl.conf +{{- end }} + +# ------------------------ 3. Tweaks for Specific Networking Configurations ----- + +#See https://help.aliyun.com/document_detail/118806.html#uicontrol-e50-ddj-w0y +sed -r -i "s@#{0,}?net.bridge.bridge-nf-call-arptables ?= ?(0|1)@net.bridge.bridge-nf-call-arptables = 1@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?vm.max_map_count ?= ?([0-9]{1,})@vm.max_map_count = 262144@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?vm.swappiness ?= ?([0-9]{1,})@vm.swappiness = 0@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?fs.inotify.max_user_instances ?= ?([0-9]{1,})@fs.inotify.max_user_instances = 524288@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?kernel.pid_max ?= ?([0-9]{1,})@kernel.pid_max = 65535@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?vm.overcommit_memory ?= ?(0|1|2)@vm.overcommit_memory = 0@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?fs.inotify.max_user_watches ?= ?([0-9]{1,})@fs.inotify.max_user_watches = 524288@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?fs.pipe-max-size ?= ?([0-9]{1,})@fs.pipe-max-size = 4194304@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.core.netdev_max_backlog ?= ?([0-9]{1,})@net.core.netdev_max_backlog = 65535@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.core.rmem_max ?= ?([0-9]{1,})@net.core.rmem_max = 33554432@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.core.wmem_max ?= ?([0-9]{1,})@net.core.wmem_max = 33554432@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.core.somaxconn ?= ?([0-9]{1,})@net.core.somaxconn = 32768@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?fs.aio-max-nr ?= ?([0-9]{1,})@fs.aio-max-nr = 262144@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?kernel.watchdog_thresh ?= ?([0-9]{1,})@kernel.watchdog_thresh = 5@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?kernel.hung_task_timeout_secs ?= ?([0-9]{1,})@kernel.hung_task_timeout_secs = 5@g" /etc/sysctl.conf +{{- if .internal_ipv4 | empty | not }} +sed -r -i "s@#{0,}?net.ipv4.tcp_tw_recycle ?= ?(0|1|2)@net.ipv4.tcp_tw_recycle = 0@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.tcp_tw_reuse ?= ?(0|1)@net.ipv4.tcp_tw_reuse = 0@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.conf.all.rp_filter ?= ?(0|1|2)@net.ipv4.conf.all.rp_filter = 1@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.conf.default.rp_filter ?= ?(0|1|2)@net.ipv4.conf.default.rp_filter = 1@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.ip_forward ?= ?(0|1)@net.ipv4.ip_forward = 1@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.bridge.bridge-nf-call-iptables ?= ?(0|1)@net.bridge.bridge-nf-call-iptables = 1@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.ip_local_reserved_ports ?= ?([0-9]{1,}-{0,1},{0,1}){1,}@net.ipv4.ip_local_reserved_ports = 30000-32767@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.tcp_max_syn_backlog ?= ?([0-9]{1,})@net.ipv4.tcp_max_syn_backlog = 1048576@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.neigh.default.gc_thresh1 ?= ?([0-9]{1,})@net.ipv4.neigh.default.gc_thresh1 = 512@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.neigh.default.gc_thresh2 ?= ?([0-9]{1,})@net.ipv4.neigh.default.gc_thresh2 = 2048@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.neigh.default.gc_thresh3 ?= ?([0-9]{1,})@net.ipv4.neigh.default.gc_thresh3 = 4096@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.conf.eth0.arp_accept ?= ?(0|1)@net.ipv4.conf.eth0.arp_accept = 1@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.tcp_retries2 ?= ?([0-9]{1,})@net.ipv4.tcp_retries2 = 15@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.tcp_max_tw_buckets ?= ?([0-9]{1,})@net.ipv4.tcp_max_tw_buckets = 1048576@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.tcp_max_orphans ?= ?([0-9]{1,})@net.ipv4.tcp_max_orphans = 65535@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.udp_rmem_min ?= ?([0-9]{1,})@net.ipv4.udp_rmem_min = 131072@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.udp_wmem_min ?= ?([0-9]{1,})@net.ipv4.udp_wmem_min = 131072@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.conf.all.arp_ignore ?= ??(0|1|2)@net.ipv4.conf.all.arp_ignore = 1@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.conf.default.arp_ignore ?= ??(0|1|2)@net.ipv4.conf.default.arp_ignore = 1@g" /etc/sysctl.conf +{{- end }} +{{- if .internal_ipv6 | empty | not }} +#add for ipv6 +sed -r -i "s@#{0,}?net.bridge.bridge-nf-call-ip6tables ?= ?(0|1)@net.bridge.bridge-nf-call-ip6tables = 1@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.conf.all.disable_ipv6 ?= ?([0-9]{1,})@net.ipv6.conf.all.disable_ipv6 = 0@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.conf.default.disable_ipv6 ?= ?([0-9]{1,})@net.ipv6.conf.default.disable_ipv6 = 0@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.conf.lo.disable_ipv6 ?= ?([0-9]{1,})@net.ipv6.conf.lo.disable_ipv6 = 0@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.conf.all.forwarding ?= ?([0-9]{1,})@net.ipv6.conf.all.forwarding = 1@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.conf.default.accept_dad ?= ?([0-9]{1,})@net.ipv6.conf.default.accept_dad = 0@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.route.max_size ?= ?([0-9]{1,})@net.ipv6.route.max_size = 65536@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.neigh.default.retrans_time_ms ?= ?([0-9]{1,})@net.ipv6.neigh.default.retrans_time_ms = 1000@g" /etc/sysctl.conf +{{- end }} + +tmpfile="$$.tmp" +awk ' !x[$0]++{print > "'$tmpfile'"}' /etc/sysctl.conf +mv $tmpfile /etc/sysctl.conf + +# ------------------------ 4. Security Limit ------------------------------------ + +# ulimit +echo "* soft nofile 1048576" >> /etc/security/limits.conf +echo "* hard nofile 1048576" >> /etc/security/limits.conf +echo "* soft nproc 65536" >> /etc/security/limits.conf +echo "* hard nproc 65536" >> /etc/security/limits.conf +echo "* soft memlock unlimited" >> /etc/security/limits.conf +echo "* hard memlock unlimited" >> /etc/security/limits.conf + +sed -r -i "s@#{0,}?\* soft nofile ?([0-9]{1,})@\* soft nofile 1048576@g" /etc/security/limits.conf +sed -r -i "s@#{0,}?\* hard nofile ?([0-9]{1,})@\* hard nofile 1048576@g" /etc/security/limits.conf +sed -r -i "s@#{0,}?\* soft nproc ?([0-9]{1,})@\* soft nproc 65536@g" /etc/security/limits.conf +sed -r -i "s@#{0,}?\* hard nproc ?([0-9]{1,})@\* hard nproc 65536@g" /etc/security/limits.conf +sed -r -i "s@#{0,}?\* soft memlock ?([0-9]{1,}([TGKM]B){0,1}|unlimited)@\* soft memlock unlimited@g" /etc/security/limits.conf +sed -r -i "s@#{0,}?\* hard memlock ?([0-9]{1,}([TGKM]B){0,1}|unlimited)@\* hard memlock unlimited@g" /etc/security/limits.conf + +tmpfile="$$.tmp" +awk ' !x[$0]++{print > "'$tmpfile'"}' /etc/security/limits.conf +mv $tmpfile /etc/security/limits.conf + +# ------------------------ 5. Firewall Configurations --------------------------- + +systemctl stop firewalld 1>/dev/null 2>/dev/null +systemctl disable firewalld 1>/dev/null 2>/dev/null +systemctl stop ufw 1>/dev/null 2>/dev/null +systemctl disable ufw 1>/dev/null 2>/dev/null + +# ------------------------ 6. System Module Settings ---------------------------- + +modinfo br_netfilter > /dev/null 2>&1 +if [ $? -eq 0 ]; then + modprobe br_netfilter + mkdir -p /etc/modules-load.d + echo 'br_netfilter' > /etc/modules-load.d/kubekey-br_netfilter.conf +fi + +modinfo overlay > /dev/null 2>&1 +if [ $? -eq 0 ]; then + modprobe overlay + echo 'overlay' >> /etc/modules-load.d/kubekey-br_netfilter.conf +fi + +# ------------------------ 7. IPTables and Connection Tracking ----------------- + +modprobe ip_vs +modprobe ip_vs_rr +modprobe ip_vs_wrr +modprobe ip_vs_sh + +cat > /etc/modules-load.d/kube_proxy-ipvs.conf </dev/null 2>/dev/null +if [ $? -eq 0 ]; then + echo 'nf_conntrack_ipv4' > /etc/modules-load.d/kube_proxy-ipvs.conf +else + modprobe nf_conntrack + echo 'nf_conntrack' > /etc/modules-load.d/kube_proxy-ipvs.conf +fi +sysctl -p + +sync +echo 3 > /proc/sys/vm/drop_caches + +# Make sure the iptables utility doesn't use the nftables backend. +{{- if .internal_ipv4 | empty | not }} +update-alternatives --set iptables /usr/sbin/iptables-legacy >/dev/null 2>&1 || true +{{- end }} +{{- if .internal_ipv6 | empty | not }} +update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy >/dev/null 2>&1 || true +{{- end }} +update-alternatives --set arptables /usr/sbin/arptables-legacy >/dev/null 2>&1 || true +update-alternatives --set ebtables /usr/sbin/ebtables-legacy >/dev/null 2>&1 || true diff --git a/builtin/core/roles/native/meta/main.yaml b/builtin/core/roles/native/meta/main.yaml new file mode 100644 index 000000000..f65e12698 --- /dev/null +++ b/builtin/core/roles/native/meta/main.yaml @@ -0,0 +1,18 @@ +dependencies: + # Initialize new Kubernetes nodes + - role: native/repository + when: + - .groups.k8s_cluster | default list | has .inventory_hostname + - .kubernetes_install_LoadState.stdout | eq "not-found" + # install nfs for nfs nodes + - role: native/nfs + when: .groups.nfs | default list | has .inventory_hostname + # Always perform initialization steps for all nodes + - role: native/ntp + when: + - .groups.k8s_cluster | default list | has .inventory_hostname + - .kubernetes_install_LoadState.stdout | eq "not-found" + - role: native/dns + - role: native/init + + diff --git a/builtin/core/roles/native/nfs/defaults/main.yaml b/builtin/core/roles/native/nfs/defaults/main.yaml new file mode 100644 index 000000000..c02abb0c4 --- /dev/null +++ b/builtin/core/roles/native/nfs/defaults/main.yaml @@ -0,0 +1,3 @@ +nfs: + share_dir: + - /share/ diff --git a/builtin/core/roles/native/nfs/tasks/main.yaml b/builtin/core/roles/native/nfs/tasks/main.yaml new file mode 100644 index 000000000..0e46ce9e9 --- /dev/null +++ b/builtin/core/roles/native/nfs/tasks/main.yaml @@ -0,0 +1,23 @@ +--- +- name: NFS | Generate NFS server export configuration + template: + src: exports + dest: /etc/exports + +- name: NFS | Ensure NFS share directories exist with correct permissions + loop: "{{ .nfs.share_dir | toJson }}" + command: | + if [ ! -d {{ .item }} ]; then + mkdir -p {{ .item }} + chmod -R 0755 {{ .item }} + chown nobody:nobody {{ .item }} + fi + +- name: NFS | Export share directories and start NFS server service + command: | + exportfs -a + {{- if .os.release.ID_LIKE | eq "debian" }} + systemctl enable nfs-kernel-server && systemctl restart nfs-kernel-server + {{- else if .os.release.ID_LIKE | unquote | eq "rhel fedora"}} + systemctl enable nfs-server.service && systemctl restart nfs-server.service + {{- end }} \ No newline at end of file diff --git a/builtin/core/roles/native/nfs/templates/exports b/builtin/core/roles/native/nfs/templates/exports new file mode 100644 index 000000000..01ae4cea1 --- /dev/null +++ b/builtin/core/roles/native/nfs/templates/exports @@ -0,0 +1,3 @@ +{{- range .nfs.share_dir }} +{{ . }} *(rw,sync,no_subtree_check) +{{- end }} diff --git a/builtin/core/roles/native/ntp/tasks/main.yaml b/builtin/core/roles/native/ntp/tasks/main.yaml new file mode 100644 index 000000000..8829c6238 --- /dev/null +++ b/builtin/core/roles/native/ntp/tasks/main.yaml @@ -0,0 +1,55 @@ +--- +- name: NTP | Configure NTP server + command: | + # Determine the correct chrony configuration file based on OS type + chronyConfigFile={{ if or (.os.release.ID | eq "ubuntu") (.os.release.ID_LIKE | eq "debian") }}"/etc/chrony/chrony.conf"{{ else }}"/etc/chrony.conf"{{ end }} + + # Remove any existing server, allow, or local entries to ensure a clean configuration + sed -i '/^server/d' $chronyConfigFile + sed -i 's/^pool /#pool /g' $chronyConfigFile + sed -i '/^allow/d' $chronyConfigFile + sed -i '/^local/d' $chronyConfigFile + + # Add base configuration to allow all clients and set local stratum + echo "allow 0.0.0.0/0" >> $chronyConfigFile + echo "allow ::/0" >> $chronyConfigFile + echo "local stratum 10" >> $chronyConfigFile + + # Add NTP server entries + {{- range $server := .native.ntp.servers }} + {{- $internalIPv4 := "" }} + {{- $internalIPv6 := "" }} + {{- range $.hostvars }} + {{- if eq .hostname $server }} + {{- $internalIPv4 = .internal_ipv4 | default "" }} + {{- $internalIPv6 = .internal_ipv6 | default "" }} + {{- end }} + {{- end }} + # Configuring NTP server: {{ $server }} + {{- if $internalIPv4 | empty | not }} + grep -q '^server {{ $internalIPv4 }} iburst' $chronyConfigFile || sed '1a server {{ $internalIPv4 }} iburst' -i $chronyConfigFile + {{- end }} + {{- if $internalIPv6 | empty | not }} + grep -q '^server {{ $internalIPv6 }} iburst' $chronyConfigFile || sed '1a server [{{ $internalIPv6 }}] iburst' -i $chronyConfigFile + {{- end }} + {{- if and ($internalIPv4 | empty) ($internalIPv6 | empty) }} + grep -q '^server {{ $server }} iburst' $chronyConfigFile || sed '1a server {{ $server }} iburst' -i $chronyConfigFile + {{- end }} + {{- end }} + when: + - .native.ntp.enabled + - .native.ntp.servers | empty | not + +- name: Timezone | Set system timezone and NTP synchronization + command: | + timedatectl set-timezone {{ .native.timezone }} + timedatectl set-ntp {{ and .native.ntp.enabled (.native.ntp.servers | empty | not) }} + when: or (and .native.ntp.enabled (.native.ntp.servers | empty | not)) (.native.timezone | empty | not) + +- name: NTP | Restart NTP service + command: | + {{- if or (.os.release.ID | eq "ubuntu") (.os.release.ID_LIKE | eq "debian") }} + systemctl restart chrony.service + {{- end }} + systemctl restart chronyd.service + when: or (and .native.ntp.enabled (.native.ntp.servers | empty | not)) (.native.timezone | empty | not) diff --git a/builtin/core/roles/native/repository/tasks/install_package.yaml b/builtin/core/roles/native/repository/tasks/install_package.yaml new file mode 100644 index 000000000..f4b4f8c87 --- /dev/null +++ b/builtin/core/roles/native/repository/tasks/install_package.yaml @@ -0,0 +1,87 @@ +- name: Repository | Check current host debian or rhel + set_fact: + current_host_type: >- + {{- if .os.release.ID_LIKE | eq "debian" }} + ubuntu + {{- else if .os.release.ID_LIKE | unquote | eq "rhel fedora" }} + centos + {{- else if .os.release.ID | unquote | eq "kylin" }} + centos + {{- end -}} + +- name: Repository | Initialize Debian-based repository and install required system packages + command: | + now=$(date +"%Y%m%d%H%M%S") + PKGS="socat conntrack ipset ebtables chrony ipvsadm{{ if .groups.nfs | default list | has .inventory_hostname }} nfs-kernel-server{{ end }}" + PKGS_TO_INSTALL="" + for pkg in $PKGS; do + if [ -n "$pkg" ]; then + dpkg -s $pkg >/dev/null 2>&1 || PKGS_TO_INSTALL="$PKGS_TO_INSTALL $pkg" + fi + done + if [ -f "{{ .tmp_dir }}/iso" ]; then + # Backup current APT sources + mv /etc/apt/sources.list /etc/apt/sources.list.kubekey-$now.bak + mv /etc/apt/sources.list.d /etc/apt/sources.list.d.kubekey-$now.bak + mkdir -p /etc/apt/sources.list.d + # Configure local repository + rm -rf /etc/apt/sources.list.d/* + echo 'deb [trusted=yes] file://{{ .tmp_dir }}/iso /' > /etc/apt/sources.list.d/kubekey.list + # Update package index + apt-get update + # Install missing packages + if [ -n "$PKGS_TO_INSTALL" ]; then + apt install -y $PKGS_TO_INSTALL + fi + # Restore original APT sources + rm -rf /etc/apt/sources.list.d + mv /etc/apt/sources.list.kubekey.bak-$now /etc/apt/sources.list + mv /etc/apt/sources.list.d.kubekey.bak-$now /etc/apt/sources.list.d + else + # No local ISO found, using default repositories + apt-get update + if [ -n "$PKGS_TO_INSTALL" ]; then + apt install -y $PKGS_TO_INSTALL + fi + fi + when: .current_host_type | trim | eq "ubuntu" + +- name: Repository | Initialize RHEL-based repository and install required system packages + command: | + BACKUP_FILE="/etc/yum.repos.d.kubekey.bak-$(date +%Y%m%d%H%M%S)" + PKGS="socat conntrack ipset ebtables chrony ipvsadm{{ if .groups.nfs | default list | has .inventory_hostname }} nfs-kernel-server{{ end }}" + PKGS_TO_INSTALL="" + for pkg in $PKGS; do + if [ -n "$pkg" ]; then + rpm -q $pkg >/dev/null 2>&1 || PKGS_TO_INSTALL="$PKGS_TO_INSTALL $pkg" + fi + done + if [ -f "{{ .tmp_dir }}/iso" ]; then + # Backup current YUM repositories + mv /etc/yum.repos.d "$BACKUP_FILE" + mkdir -p /etc/yum.repos.d + # Configure local repository + rm -rf /etc/yum.repos.d/* + cat < /etc/yum.repos.d/CentOS-local.repo + [base-local] + name=Local RPM Repository + baseurl=file://{{ .tmp_dir }}/iso + enabled=1 + gpgcheck=0 + EOF + # Refresh repository cache + yum clean all && yum makecache + # Install missing packages + if [ -n "$PKGS_TO_INSTALL" ]; then + yum install -y $PKGS_TO_INSTALL + fi + # Restore original YUM repositories + rm -rf /etc/yum.repos.d + mv "$BACKUP_FILE" /etc/yum.repos.d + else + # No local ISO found, using default repositories + if [ -n "$PKGS_TO_INSTALL" ]; then + yum install -y $PKGS_TO_INSTALL + fi + fi + when: .current_host_type | trim | eq "centos" \ No newline at end of file diff --git a/builtin/core/roles/native/repository/tasks/main.yaml b/builtin/core/roles/native/repository/tasks/main.yaml new file mode 100644 index 000000000..da1823046 --- /dev/null +++ b/builtin/core/roles/native/repository/tasks/main.yaml @@ -0,0 +1,65 @@ +--- +- name: Repository | Synchronize local repository ISO image + block: + - name: Repository | Prepare tmp files + command: | + mkdir -p {{ .tmp_dir }}/iso + + - name: Repository | Check system version when use Kylin + set_fact: + sp_version: >- + {{- if .os.release.VERSION | contains "Tercel" }} + SP1 + {{- else if .os.release.VERSION | contains "Sword" }} + SP2 + {{- else if .os.release.VERSION | contains "Lance" }} + SP3 + {{- end -}} + when: .os.release.ID | unquote | eq "kylin" + + - name: Repository | Define the system string based on distribution + set_fact: + system_string: >- + {{- if .os.release.ID | unquote | eq "kylin" }} + kylin-{{ .os.release.VERSION_ID }}-{{ .sp_version }} + {{- else if .os.release.ID_LIKE | unquote | eq "rhel fedora" }} + {{ .os.release.ID }}{{ .os.release.VERSION_ID }} + {{- else }} + {{ .os.release.ID }}-{{ .os.release.VERSION_ID }} + {{- end -}} + + - name: Repository | Define the package file type by system info + set_fact: + iso_type: >- + {{- if .os.release.ID_LIKE | eq "debian" }} + debs + {{- else }} + rpms + {{- end -}} + + - name: Repository | Set iso file name + when: + - .iso_name | empty + set_fact: + iso_name: "{{ .system_string | unquote | trim | lower }}-{{ .iso_type | trim }}-{{ .binary_type }}.iso" + + - name: Repository | Copy local repository ISO file + ignore_errors: true + copy: + src: >- + {{ .binary_dir }}/repository/{{ .iso_name }} + dest: >- + {{ .tmp_dir }}/repository.iso + - name: Repository | Mount repository ISO to temporary directory + command: | + if [ -f "{{ .tmp_dir }}/repository.iso" ]; then + mount -t iso9660 -o loop {{ .tmp_dir }}/repository.iso {{ .tmp_dir }}/iso + fi + - name: Repository | Initialize package repositories and install system dependencies + include_tasks: install_package.yaml + always: + - name: Repository | Unmount repository ISO from temporary directory + command: | + if [ -f "{{ .tmp_dir }}/repository.iso" ]; then + umount {{ .tmp_dir }}/iso + fi \ No newline at end of file diff --git a/builtin/core/roles/native/root/tasks/main.yaml b/builtin/core/roles/native/root/tasks/main.yaml new file mode 100644 index 000000000..20d4b5457 --- /dev/null +++ b/builtin/core/roles/native/root/tasks/main.yaml @@ -0,0 +1,54 @@ +- name: OS | Add sudo secure path + tags: ["always"] + command: | + ADD_PATHS="/usr/local/bin" + BACKUP_FILE="/etc/sudoers.backup.$(date +%Y%m%d_%H%M%S)" + cp -p /etc/sudoers "$BACKUP_FILE" + TMP_FILE=$(mktemp /tmp/sudoers_update.XXXXXX) + chmod 600 "$TMP_FILE" + cp -p /etc/sudoers "$TMP_FILE" + cleanup() { + rm -f "$TMP_FILE" + rm -f "$BACKUP_FILE" + } + trap cleanup EXIT INT TERM + if grep -q "^Defaults.*secure_path" "$TMP_FILE"; then + echo "find current secure_path 配置" + EXISTING_LINE=$(grep "^Defaults.*secure_path" "$TMP_FILE") + EXISTING_PATH=$(echo "$EXISTING_LINE" | sed -e 's/.*secure_path[[:space:]]*=[[:space:]]*"\{0,1\}\([^"[:space:]]*\)"\{0,1\}.*/\1/') + if [ -n "$EXISTING_PATH" ]; then + echo "current secure_path: $EXISTING_PATH" + NEW_PATH="$EXISTING_PATH" + IFS=':' read -ra PATHS_TO_ADD <<< "$ADD_PATHS" + for path in "${PATHS_TO_ADD[@]}"; do + if [[ ":$NEW_PATH:" != *":$path:"* ]]; then + NEW_PATH="$NEW_PATH:$path" + fi + done + echo "new secure_path: $NEW_PATH" + sed -i "s/^Defaults.*secure_path/# &/" "$TMP_FILE" + echo "Defaults secure_path=\"$NEW_PATH\"" >> "$TMP_FILE" + else + echo "warning: can not get current secure_path" + echo "Defaults secure_path=\"$ADD_PATHS\"" >> "$TMP_FILE" + fi + else + echo "current secure_path config not found,set new data" + echo "Defaults secure_path=\"$ADD_PATHS\"" >> "$TMP_FILE" + fi + if /usr/sbin/visudo -cf "$TMP_FILE" > /dev/null 2>&1; then + cp -f "$TMP_FILE" /etc/sudoers + chmod 440 /etc/sudoers + echo "already update /etc/sudoers" + echo "after update secure_path config:" + grep "^Defaults.*secure_path" /etc/sudoers + else + echo "error: something went wrong,roll back" + echo "please check visudo log:" + /usr/sbin/visudo -cf "$TMP_FILE" + cp -f "$BACKUP_FILE" /etc/sudoers + chmod 440 /etc/sudoers + echo "already roll back" + exit 1 + fi + echo "success" \ No newline at end of file diff --git a/builtin/core/roles/precheck/artifact/tasks/main.yaml b/builtin/core/roles/precheck/artifact/tasks/main.yaml new file mode 100644 index 000000000..9ae35fb3a --- /dev/null +++ b/builtin/core/roles/precheck/artifact/tasks/main.yaml @@ -0,0 +1,29 @@ +--- +- name: Artifact | Ensure artifact file exists + when: .download.artifact_file | empty | not + command: | + if [ ! -f "{{ .download.artifact_file }}" ]; then + echo "Error: Artifact file '{{ .download.artifact_file }}' does not exist." + exit 1 + fi + +- name: Artifact | Validate artifact file extension + when: + - .download.artifact_file | empty | not + loop: ['.tgz','.tar.gz'] + command: | + if [[ "{{ .download.artifact_file }}" != *{{ .item }} ]]; then + echo "Error: Artifact file '{{ .download.artifact_file }}' does not have the required extension '{{ .item }}'." + exit 1 + fi + +- name: Artifact | Verify artifact MD5 checksum + when: + - .download.artifact_md5 | empty | not + - .download.artifact_file | empty | not + command: | + actual_md5=$(md5sum {{ .download.artifact_file }} | awk '{print $1}') + if [[ "$actual_md5" != "{{ .download.artifact_md5 }}" ]]; then + echo "Error: MD5 checksum mismatch for '{{ .download.artifact_file }}'. Expected '{{ .download.artifact_md5 }}', got '$actual_md5'." + exit 1 + fi diff --git a/builtin/core/roles/precheck/cri/tasks/main.yaml b/builtin/core/roles/precheck/cri/tasks/main.yaml new file mode 100644 index 000000000..5454e6783 --- /dev/null +++ b/builtin/core/roles/precheck/cri/tasks/main.yaml @@ -0,0 +1,17 @@ +--- +- name: CRI | Fail if container manager is not docker or containerd + run_once: true + assert: + that: .cluster_require.require_container_manager | has .cri.container_manager + fail_msg: >- + The specified container manager "{{ .cri.container_manager }}" is not supported. Please use one of the following: {{ .cluster_require.require_container_manager | toJson }}. + +- name: CRI | Validate minimum required containerd version + run_once: true + when: + - .cri.containerd_version | empty | not + - .cri.container_manager | eq "containerd" + assert: + that: .cri.containerd_version | semverCompare (printf ">=%s" .cluster_require.containerd_min_version_required) + fail_msg: >- + The detected containerd version ({{ .containerd_version }}) is below the minimum required version: {{ .cluster_require.containerd_min_version_required }}. diff --git a/builtin/core/roles/precheck/etcd/tasks/main.yaml b/builtin/core/roles/precheck/etcd/tasks/main.yaml new file mode 100644 index 000000000..99d890cc7 --- /dev/null +++ b/builtin/core/roles/precheck/etcd/tasks/main.yaml @@ -0,0 +1,34 @@ +--- +- name: Kubernetes | Fail if etcd deployment type is not "internal" or "external" + run_once: true + assert: + that: .cluster_require.require_etcd_deployment_type | has .etcd.deployment_type + fail_msg: >- + Invalid etcd deployment type: "{{ .etcd.deployment_type }}". Expected "internal" or "external". + +## https://cwiki.yunify.com/pages/viewpage.action?pageId=145920824 +- name: ETCD | Validate disk I/O performance for etcd + when: + - .groups.etcd | default list | has .inventory_hostname + block: + - name: ETCD | Check if fio is installed + ignore_errors: true + command: fio --version + register: fio_install_version + - name: ETCD | Run fio disk I/O test + when: .fio_install_version.error | empty + block: + - name: ETCD | Execute fio and collect results + command: | + mkdir -p {{ .tmp_dir }}/etcd/test-data + fio --rw=write --ioengine=sync --fdatasync=1 --directory={{ .tmp_dir }}/etcd/test-data --size=22m --bs=2300 --name=mytest --output-format=json + register: fio_result + register_type: json + - name: ETCD | Assert disk fsync latency meets requirements + assert: + that: (index (.fio_result.stdout.jobs | first) "sync" "lat_ns" "percentile" "90.000000") | le .cluster_require.etcd_disk_wal_fysnc_duration_seconds + fail_msg: >- + The 90th percentile fsync latency is {{ index (.fio_result.stdout.jobs | first) "sync" "lat_ns" "percentile" "90.000000" }}ns, which exceeds the maximum allowed: {{ .cluster_require.etcd_disk_wal_fysnc_duration_seconds }}ns. + always: + - name: ETCD | Clean up fio test data directory + command: rm -rf {{ .tmp_dir }}/etcd/test-data diff --git a/builtin/core/roles/precheck/inventory/tasks/main.yaml b/builtin/core/roles/precheck/inventory/tasks/main.yaml new file mode 100644 index 000000000..c884b0fac --- /dev/null +++ b/builtin/core/roles/precheck/inventory/tasks/main.yaml @@ -0,0 +1,21 @@ +- name: Inventory | Kubernetes groups must not be empty + when: .kubernetes.kube_version | empty | not + assert: + that: + - .groups.k8s_cluster | empty | not + - .groups.kube_control_plane | empty | not + fail_msg: >- + The Kubernetes inventory groups "k8s_cluster" and "kube_control_plane" must not be empty. Please ensure both groups are present and correctly specified in your inventory file. + +- name: Inventory | etcd group must not be empty + when: .kubernetes.kube_version | empty | not + assert: + that: .groups.etcd | empty | not + fail_msg: >- + The etcd inventory group "etcd" must not be empty. Please verify that this group is properly defined in your inventory. + +- name: Inventory | Fail if the number of etcd hosts is even + assert: + that: (mod (.groups.etcd | len) 2) | eq 1 + fail_msg: >- + The number of etcd nodes must be odd to maintain quorum. The current count is {{ .groups.etcd | len }}. Please adjust your inventory so that the etcd group contains an odd number of hosts. diff --git a/builtin/core/roles/precheck/kubernetes/tasks/main.yaml b/builtin/core/roles/precheck/kubernetes/tasks/main.yaml new file mode 100644 index 000000000..2e057e2da --- /dev/null +++ b/builtin/core/roles/precheck/kubernetes/tasks/main.yaml @@ -0,0 +1,56 @@ +- name: Kubernetes | Validate kube-vip address + run_once: true + when: .kubernetes.control_plane_endpoint.type | eq "kube_vip" + assert: + that: + - .kubernetes.control_plane_endpoint.kube_vip.address | empty | not + - .kubernetes.control_plane_endpoint.kube_vip.address | regexMatch "^((25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])|(([0-9a-fA-F]{1,4}:){7}([0-9a-fA-F]{1,4}|:)|(([0-9a-fA-F]{1,4}:){1,6}|:):([0-9a-fA-F]{1,4}|:){1,6}([0-9a-fA-F]{1,4}|:)))$" + - | + {{- $existIP := false }} + {{- range .groups.all | default list }} + {{- if or ($.kubernetes.control_plane_endpoint.kube_vip.address | eq (index $.hostvars . "internal_ipv4" | default "")) ($.kubernetes.control_plane_endpoint.kube_vip.address | eq (index $.hostvars . "internal_ipv6" | default "")) }} + {{- $existIP = true }} + {{- end }} + {{- end }} + {{ not $existIP }} + fail_msg: >- + The value of "kubernetes.control_plane_endpoint.kube_vip.address" must be an IP address that is not currently assigned to any node. + +- name: Kubernetes | Fail if unsupported Kubernetes version + run_once: true + assert: + that: + - .kubernetes.kube_version | empty | not + - .kubernetes.kube_version | semverCompare (printf ">=%s" .cluster_require.kube_version_min_required) + fail_msg: >- + This version of KubeKey only supports Kubernetes versions greater than or equal to {{ .cluster_require.kube_version_min_required }}. You are attempting to use version {{ .kube_version }}. + +- name: Kubernetes | Check if Kubernetes is installed + block: + - name: Kubernetes | Validate Kubernetes service status and version + when: .kubernetes_install_LoadState.stdout | eq "loaded" + block: + - name: Kubernetes | Ensure kubelet service is active + assert: + that: .kubernetes_install_ActiveState.stdout | eq "active" + fail_msg: >- + The kubelet service must be running and active when it is loaded. + - name: Kubernetes | Ensure installed Kubernetes version matches expected version + assert: + that: .kubernetes_install_version.stdout | default "" | trimPrefix "Kubernetes " | eq .kube_version + fail_msg: >- + The installed Kubernetes version ({{ .kubernetes_install_version.stdout | default "" | trimPrefix "Kubernetes " }}) does not match the expected version ({{ .kube_version }}). + +- name: ImageRegistry | Verify successful authentication to image registry + when: + - .image_registry.auth.registry | empty | not + - .image_registry.type | empty + run_once: true + command: | + HTTP_CODE=$(curl -skLI -w "%{http_code}" -u "{{ .image_registry.auth.username }}:{{ .image_registry.auth.password }}" "https://{{ .image_registry.auth.registry }}/v2/" -o /dev/null) + if [[ "$HTTP_CODE" == "200" ]]; then + echo "Successfully authenticated to the image registry." + else + echo "Failed to authenticate to the image registry at {{ .image_registry.auth.registry }}." >&2 + exit 1 + fi diff --git a/builtin/core/roles/precheck/meta/main.yaml b/builtin/core/roles/precheck/meta/main.yaml new file mode 100644 index 000000000..da5249f91 --- /dev/null +++ b/builtin/core/roles/precheck/meta/main.yaml @@ -0,0 +1,32 @@ +dependencies: + - role: precheck/inventory + when: .inventory_hostname | eq "localhost" + + - role: precheck/artifact + tags: ["artifact"] + when: .inventory_hostname | eq "localhost" + + - role: precheck/kubernetes + tags: ["kubernetes"] + when: .groups.k8s_cluster | default list | has .inventory_hostname + + - role: precheck/etcd + tags: ["etcd"] + when: .groups.etcd | default list | has .inventory_hostname + + - role: precheck/os + tags: ["os"] + when: .inventory_hostname | ne "localhost" + + - role: precheck/network + tags: ["network"] + when: .inventory_hostname | ne "localhost" + + - role: precheck/cri + tags: ["cri"] + when: .groups.k8s_cluster | default list | has .inventory_hostname + + - role: precheck/nfs + tags: ["nfs"] + when: .groups.nfs | default list | has .inventory_hostname + diff --git a/builtin/core/roles/precheck/network/tasks/main.yaml b/builtin/core/roles/precheck/network/tasks/main.yaml new file mode 100644 index 000000000..54f8b218b --- /dev/null +++ b/builtin/core/roles/precheck/network/tasks/main.yaml @@ -0,0 +1,97 @@ +--- +- name: Network | Ensure either internal_ipv4 or internal_ipv6 is defined + assert: + that: or (.internal_ipv4 | empty | not) (.internal_ipv6 | empty | not) + fail_msg: >- + Either "internal_ipv4" or "internal_ipv6" must be specified. Both cannot be empty. + +- name: Network | Ensure required network interfaces are present + command: | + {{- if .internal_ipv4 | empty | not }} + if ! ip -o addr show | grep -q {{ .internal_ipv4 }}; then + echo 'The specified IPv4 address is not assigned to any network interface.' >&2 + exit 1 + fi + {{- end }} + {{- if .internal_ipv6 | empty | not }} + if ! ip -o addr show | grep -q {{ .internal_ipv6 }}; then + echo 'The specified IPv6 address is not assigned to any network interface.' >&2 + exit 1 + fi + {{- end }} + +# https://kubernetes.io/docs/concepts/services-networking/dual-stack/ +- name: Network | Validate dual-stack CIDR configuration + run_once: true + block: + - name: Network | Check pod CIDR includes both IPv4 and IPv6 + when: .cni.pod_cidr | empty | not + assert: + that: .cni.pod_cidr | splitList "," | len | ge 2 + fail_msg: >- + "cni.pod_cidr" must specify both IPv4 and IPv6 ranges, using either the format ipv4_cidr/ipv6_cidr or ipv4_cidr,ipv6_cidr. + - name: Network | Check service CIDR includes both IPv4 and IPv6 + when: .cni.service_cidr | empty | not + assert: + that: .cni.service_cidr | splitList "," | len | ge 2 + fail_msg: >- + "cni.service_cidr" must specify both IPv4 and IPv6 ranges, using either the format ipv4_cidr/ipv6_cidr or ipv4_cidr,ipv6_cidr. + - name: Network | Ensure pod networking is properly configured for dual-stack + when: + - .cni.pod_cidr | empty | not + - .cni.pod_cidr | splitList "," | len | eq 2 + assert: + that: + - .kubernetes.kube_version | semverCompare ">=v1.20.0" + - .cni.pod_cidr | splitList "," | first | ipFamily | eq "IPv4" + - .cni.pod_cidr | splitList "," | last | ipFamily | eq "IPv6" + fail_msg: >- + Dual-stack pod networking is only supported in Kubernetes v1.20.0 or newer. + - name: Network | Ensure service networking is properly configured for dual-stack + when: + - .cni.service_cidr | empty | not + - .cni.service_cidr | splitList "," | len | eq 2 + assert: + that: + - .kubernetes.kube_version | semverCompare ">=v1.20.0" + - .cni.service_cidr | splitList "," | first | ipFamily | eq "IPv4" + - .cni.service_cidr | splitList "," | last | ipFamily | eq "IPv6" + fail_msg: >- + Dual-stack service networking is only supported in Kubernetes v1.20.0 or newer. + +- name: Network | Fail if the selected network plugin is not supported + run_once: true + when: .cni.type | empty | not + assert: + that: .cluster_require.require_network_plugin | has .cni.type + fail_msg: >- + The network plugin "{{ .cni.type }}" is not supported. Please select a supported network plugin. + +# Note: This check is intentionally conservative. While it is technically possible to schedule more pods than the available addresses in the CIDR range (for example, if some pods use the host network), this cannot be reliably determined at provisioning time. This check ensures there is enough address space for the configured maximum pods per node. +# Note: IPv6-only scenarios are not checked here. +- name: Network | Ensure sufficient address space for all pods + run_once: true + when: .groups.k8s_cluster | default list | has .inventory_hostname + block: + - name: Network | Ensure enough IPv4 addresses are available for pods + when: .cni.pod_cidr | default "10.233.64.0/18" | splitList "," | first | ipFamily | eq "IPv4" + assert: + that: le (.cni.max_pods | default 110) (sub (pow 2 (float64 (sub 32 (.cni.ipv4_mask_size | default 24)))) 2) + fail_msg: >- + The configured maximum number of pods per node exceeds the number of available IPv4 addresses in the pod CIDR range. + - name: Network | Ensure enough IPv6 addresses are available for pods + when: .cni.pod_cidr | default "10.233.64.0/18" | splitList "," | last | ipFamily | eq "IPv6" + assert: + that: le (.cni.max_pods | default 110) (sub (pow 2 (float64 (sub 128 (.cni.ipv4_mask_size | default 64)))) 2) + fail_msg: >- + The configured maximum number of pods per node exceeds the number of available IPv6 addresses in the pod CIDR range. + +# https://github.com/alibaba/hybridnet/wiki/Getting-Started#install +- name: Network | Fail if Kubernetes version is too old for hybridnet + run_once: true + assert: + that: .kubernetes.kube_version | semverCompare ">=v1.16.0" + fail_msg: >- + Hybridnet requires Kubernetes version 1.16.0 or newer. + when: + - .cni.type | eq "hybridnet" \ No newline at end of file diff --git a/builtin/core/roles/precheck/nfs/tasks/main.yaml b/builtin/core/roles/precheck/nfs/tasks/main.yaml new file mode 100644 index 000000000..3beb8c9e4 --- /dev/null +++ b/builtin/core/roles/precheck/nfs/tasks/main.yaml @@ -0,0 +1,5 @@ +--- +- name: NFS | Fail if more than one NFS server is defined + assert: + that: .groups.nfs | default list | len | eq 1 + fail_msg: "Exactly one NFS server must be specified. Multiple NFS servers are not supported." diff --git a/builtin/core/roles/precheck/os/tasks/main.yaml b/builtin/core/roles/precheck/os/tasks/main.yaml new file mode 100644 index 000000000..be2ae4b1f --- /dev/null +++ b/builtin/core/roles/precheck/os/tasks/main.yaml @@ -0,0 +1,35 @@ +--- +- name: OS | Fail if hostname is invalid + assert: + that: .hostname | regexMatch "^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*$" + fail_msg: >- + The hostname "{{ .hostname }}" is invalid. Hostnames must use only lowercase alphanumeric characters, '.', or '-', and must start and end with an alphanumeric character. + +- name: OS | Fail if operating system is not supported + assert: + that: or (.cluster_require.allow_unsupported_distribution_setup) (.cluster_require.supported_os_distributions | has .os.release.ID) + fail_msg: >- + The operating system "{{ .os.release.ID }}" is not recognized or supported. + +- name: OS | Fail if architecture is not supported + assert: + that: .cluster_require.supported_architectures | has .os.architecture + fail_msg: >- + The system architecture "{{ .os.architecture }}" is not supported. + +- name: OS | Fail if master node memory is insufficient + assert: + that: .process.memInfo.MemTotal | trimSuffix " kB" | atoi | le .cluster_require.minimal_master_memory_mb + when: .groups.kube_control_plane | default list | has .inventory_hostname + +- name: OS | Fail if worker node memory is insufficient + assert: + that: .process.memInfo.MemTotal | trimSuffix " kB" | atoi | le .cluster_require.minimal_node_memory_mb + when: + - .groups.kube_worker | default list | has .inventory_hostname + +- name: OS | Fail if kernel version is too old + assert: + that: .os.kernel_version | splitList "-" | first | semverCompare (printf ">=%s" .cluster_require.min_kernel_version) + fail_msg: >- + The kernel version "{{ .os.kernel_version }}" is too old. Minimum required version: {{ .cluster_require.min_kernel_version }}. diff --git a/builtin/core/roles/security/tasks/main.yaml b/builtin/core/roles/security/tasks/main.yaml new file mode 100644 index 000000000..e549883e5 --- /dev/null +++ b/builtin/core/roles/security/tasks/main.yaml @@ -0,0 +1,39 @@ +--- +- name: Security | Enhance etcd node security permissions + command: | + chmod 700 /etc/ssl/etcd/ssl && chown root:root /etc/ssl/etcd/ssl + chmod 600 /etc/ssl/etcd/ssl/* && chown root:root /etc/ssl/etcd/ssl/* + chmod 700 /var/lib/etcd && chown etcd:etcd /var/lib/etcd + chmod 550 /usr/local/bin/etcd* && chown root:root /usr/local/bin/etcd* + when: .groups.etcd | default list | has .inventory_hostname + +- name: Security | Apply security best practices for control plane nodes + command: | + chmod 644 /etc/kubernetes && chown root:root /etc/kubernetes + chmod 600 -R /etc/kubernetes && chown root:root -R /etc/kubernetes/* + chmod 644 /etc/kubernetes/manifests && chown root:root /etc/kubernetes/manifests + chmod 644 /etc/kubernetes/pki && chown root:root /etc/kubernetes/pki + chmod 600 -R /etc/cni/net.d && chown root:root -R /etc/cni/net.d + chmod 550 /usr/local/bin/ && chown root:root /usr/local/bin/ + chmod 550 -R /usr/local/bin/kube* && chown root:root -R /usr/local/bin/kube* + chmod 550 /usr/local/bin/helm && chown root:root /usr/local/bin/helm + chmod 550 -R /opt/cni/bin && chown root:root -R /opt/cni/bin + chmod 640 /var/lib/kubelet/config.yaml && chown root:root /var/lib/kubelet/config.yaml + chmod 640 -R /etc/systemd/system/kubelet.service* && chown root:root -R /etc/systemd/system/kubelet.service* + chmod 640 /etc/systemd/system/k8s-certs-renew* && chown root:root /etc/systemd/system/k8s-certs-renew* + when: .groups.kube_control_plane | default list | has .inventory_hostname + +- name: Security | Apply security best practices for worker nodes + command: | + chmod 644 /etc/kubernetes && chown root:root /etc/kubernetes + chmod 600 -R /etc/kubernetes && chown root:root -R /etc/kubernetes/* + chmod 644 /etc/kubernetes/manifests && chown root:root /etc/kubernetes/manifests + chmod 644 /etc/kubernetes/pki && chown root:root /etc/kubernetes/pki + chmod 600 -R /etc/cni/net.d && chown root:root -R /etc/cni/net.d + chmod 550 /usr/local/bin/ && chown root:root /usr/local/bin/ + chmod 550 -R /usr/local/bin/kube* && chown root:root -R /usr/local/bin/kube* + chmod 550 /usr/local/bin/helm && chown root:root /usr/local/bin/helm + chmod 550 -R /opt/cni/bin && chown root:root -R /opt/cni/bin + chmod 640 /var/lib/kubelet/config.yaml && chown root:root /var/lib/kubelet/config.yaml + chmod 640 -R /etc/systemd/system/kubelet.service* && chown root:root -R /etc/systemd/system/kubelet.service* + when: .groups.kube_worker | default list | has .inventory_hostname diff --git a/builtin/core/roles/storage-class/local/tasks/main.yaml b/builtin/core/roles/storage-class/local/tasks/main.yaml new file mode 100644 index 000000000..ea63ec159 --- /dev/null +++ b/builtin/core/roles/storage-class/local/tasks/main.yaml @@ -0,0 +1,9 @@ +--- +- name: Local | Generate the local storage manifest + template: + src: local-volume.yaml + dest: /etc/kubernetes/addons/local-volume.yaml + +- name: Local | Deploy the local storage manifest + command: | + kubectl apply -f /etc/kubernetes/addons/local-volume.yaml diff --git a/builtin/core/roles/storage-class/local/templates/local-volume.yaml b/builtin/core/roles/storage-class/local/templates/local-volume.yaml new file mode 100644 index 000000000..5ddb0ced9 --- /dev/null +++ b/builtin/core/roles/storage-class/local/templates/local-volume.yaml @@ -0,0 +1,153 @@ +--- +#Sample storage classes for OpenEBS Local PV +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: local + annotations: + storageclass.kubesphere.io/supported-access-modes: '["ReadWriteOnce"]' + storageclass.beta.kubernetes.io/is-default-class: "{{ if .storage_class.local.default }}true{{ else }}false{{ end }}" + openebs.io/cas-type: local + cas.openebs.io/config: | + - name: StorageType + value: "hostpath" + - name: BasePath + value: "{{ .storage_class.local.path }}" +provisioner: openebs.io/local +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Delete +--- +# Create Maya Service Account +apiVersion: v1 +kind: ServiceAccount +metadata: + name: openebs-maya-operator + namespace: kube-system +--- +# Define Role that allows operations on K8s pods/deployments +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-maya-operator +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["*"] + - apiGroups: ["*"] + resources: ["nodes", "nodes/proxy"] + verbs: ["*"] + - apiGroups: ["*"] + resources: ["namespaces", "services", "pods", "pods/exec", "deployments", "deployments/finalizers", "replicationcontrollers", "replicasets", "events", "endpoints", "configmaps", "secrets", "jobs", "cronjobs"] + verbs: ["*"] + - apiGroups: ["*"] + resources: ["statefulsets", "daemonsets"] + verbs: ["*"] + - apiGroups: ["*"] + resources: ["resourcequotas", "limitranges"] + verbs: ["list", "watch"] + - apiGroups: ["*"] + resources: ["ingresses", "horizontalpodautoscalers", "verticalpodautoscalers", "poddisruptionbudgets", "certificatesigningrequests"] + verbs: ["list", "watch"] + - apiGroups: ["*"] + resources: ["storageclasses", "persistentvolumeclaims", "persistentvolumes"] + verbs: ["*"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: [ "get", "list", "create", "update", "delete", "patch"] + - apiGroups: ["openebs.io"] + resources: [ "*"] + verbs: ["*"] + - nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +# Bind the Service Account with the Role Privileges. +# TODO: Check if default account also needs to be there +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-maya-operator +subjects: + - kind: ServiceAccount + name: openebs-maya-operator + namespace: kube-system +roleRef: + kind: ClusterRole + name: openebs-maya-operator + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: openebs-localpv-provisioner + namespace: kube-system + labels: + name: openebs-localpv-provisioner + openebs.io/component-name: openebs-localpv-provisioner + openebs.io/version: 3.3.0 +spec: + selector: + matchLabels: + name: openebs-localpv-provisioner + openebs.io/component-name: openebs-localpv-provisioner + replicas: 1 + strategy: + type: Recreate + template: + metadata: + labels: + name: openebs-localpv-provisioner + openebs.io/component-name: openebs-localpv-provisioner + openebs.io/version: 3.3.0 + spec: + serviceAccountName: openebs-maya-operator + containers: + - name: openebs-provisioner-hostpath + imagePullPolicy: IfNotPresent + image: {{ .storage_class.local.provisioner_image.registry }}/{{ .storage_class.local.provisioner_image.repository }}:{{ .storage_class.local.provisioner_image.tag }} + env: + # OPENEBS_IO_K8S_MASTER enables openebs provisioner to connect to K8s + # based on this address. This is ignored if empty. + # This is supported for openebs provisioner version 0.5.2 onwards + #- name: OPENEBS_IO_K8S_MASTER + # value: "http://10.128.0.12:8080" + # OPENEBS_IO_KUBE_CONFIG enables openebs provisioner to connect to K8s + # based on this config. This is ignored if empty. + # This is supported for openebs provisioner version 0.5.2 onwards + #- name: OPENEBS_IO_KUBE_CONFIG + # value: "/home/ubuntu/.kube/config" + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OPENEBS_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + # OPENEBS_SERVICE_ACCOUNT provides the service account of this pod as + # environment variable + - name: OPENEBS_SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: OPENEBS_IO_ENABLE_ANALYTICS + value: "true" + - name: OPENEBS_IO_INSTALLER_TYPE + value: "openebs-operator-lite" + - name: OPENEBS_IO_HELPER_IMAGE + value: "{{ .storage_class.local.linux_utils_image.registry }}/{{ .storage_class.local.linux_utils_image.repository }}:{{ .storage_class.local.linux_utils_image.tag }}" + # LEADER_ELECTION_ENABLED is used to enable/disable leader election. By default + # leader election is enabled. + #- name: LEADER_ELECTION_ENABLED + # value: "true" + # OPENEBS_IO_IMAGE_PULL_SECRETS environment variable is used to pass the image pull secrets + # to the helper pod launched by local-pv hostpath provisioner + #- name: OPENEBS_IO_IMAGE_PULL_SECRETS + # value: "" + livenessProbe: + exec: + command: + - sh + - -c + - test $(pgrep -c "^provisioner-loc.*") = 1 + initialDelaySeconds: 30 + periodSeconds: 60 diff --git a/builtin/core/roles/storage-class/meta/main.yaml b/builtin/core/roles/storage-class/meta/main.yaml new file mode 100644 index 000000000..b1a57dac2 --- /dev/null +++ b/builtin/core/roles/storage-class/meta/main.yaml @@ -0,0 +1,7 @@ +--- +dependencies: + - role: storage-class/local + when: .storage_class.local.enabled + + - role: storage-class/nfs + when: .storage_class.nfs.enabled diff --git a/builtin/core/roles/storage-class/nfs/tasks/main.yaml b/builtin/core/roles/storage-class/nfs/tasks/main.yaml new file mode 100644 index 000000000..a3f4300b6 --- /dev/null +++ b/builtin/core/roles/storage-class/nfs/tasks/main.yaml @@ -0,0 +1,13 @@ +--- +- name: NFS | Synchronize NFS provisioner Helm chart to remote host + copy: + src: >- + {{ .work_dir }}/kubekey/sc/nfs-subdir-external-provisioner-{{ .storage_class.nfs_provisioner_version }}.tgz + dest: >- + /etc/kubernetes/addons/nfs-subdir-external-provisioner-{{ .storage_class.nfs_provisioner_version }}.tgz + +- name: NFS | Deploy the NFS provisioner using Helm + command: | + helm upgrade --install nfs-subdir-external-provisioner /etc/kubernetes/addons/nfs-subdir-external-provisioner-{{ .nfs_provisioner_version }}.tgz --namespace kube-system \ + --set nfs.server={{ .storage_class.nfs.server }} --set nfs.path={{ .storage_class.nfs.path }} \ + --set storageClass.defaultClass={{ if .storage_class.local.default }}true{{ else }}false{{ end }} diff --git a/builtin/core/roles/uninstall/cri/containerd/defaults/main.yaml b/builtin/core/roles/uninstall/cri/containerd/defaults/main.yaml new file mode 100644 index 000000000..ba5501cef --- /dev/null +++ b/builtin/core/roles/uninstall/cri/containerd/defaults/main.yaml @@ -0,0 +1,3 @@ +cri: + containerd: + data_root: /var/lib/containerd \ No newline at end of file diff --git a/builtin/core/roles/uninstall/cri/containerd/tasks/main.yaml b/builtin/core/roles/uninstall/cri/containerd/tasks/main.yaml new file mode 100644 index 000000000..928d28256 --- /dev/null +++ b/builtin/core/roles/uninstall/cri/containerd/tasks/main.yaml @@ -0,0 +1,17 @@ +--- +- name: Containerd | Uninstall the containerd system service + ignore_errors: true + command: | + systemctl stop containerd.service + systemctl disable containerd.service + rm -rf /etc/systemd/system/containerd.service* + systemctl daemon-reload + systemctl reset-failed containerd.service + +- name: Containerd | Remove all containerd-related files and binaries + command: | + rm -rf {{ .cri.containerd.data_root }} + rm -rf /etc/containerd + rm -rf /usr/local/bin/containerd* + rm -f /usr/local/bin/runc + rm -f /usr/local/bin/ctr diff --git a/builtin/core/roles/uninstall/cri/crictl/tasks/main.yaml b/builtin/core/roles/uninstall/cri/crictl/tasks/main.yaml new file mode 100644 index 000000000..06252bcdb --- /dev/null +++ b/builtin/core/roles/uninstall/cri/crictl/tasks/main.yaml @@ -0,0 +1,3 @@ +- name: Crictl | Remove crictl binary and clean up any residual files + command: | + rm -f /usr/local/bin/crictl \ No newline at end of file diff --git a/builtin/core/roles/uninstall/cri/docker/defaults/main.yaml b/builtin/core/roles/uninstall/cri/docker/defaults/main.yaml new file mode 100644 index 000000000..888b6f795 --- /dev/null +++ b/builtin/core/roles/uninstall/cri/docker/defaults/main.yaml @@ -0,0 +1,3 @@ +cri: + docker: + data_root: /var/lib/docker \ No newline at end of file diff --git a/builtin/core/roles/uninstall/cri/docker/tasks/cridockerd.yaml b/builtin/core/roles/uninstall/cri/docker/tasks/cridockerd.yaml new file mode 100644 index 000000000..7447024af --- /dev/null +++ b/builtin/core/roles/uninstall/cri/docker/tasks/cridockerd.yaml @@ -0,0 +1,14 @@ +--- +- name: Cridockerd | Stop and disable the cri-dockerd system service + ignore_errors: true + command: | + systemctl stop cri-dockerd.service + systemctl disable cri-dockerd.service + rm -rf /etc/systemd/system/cri-dockerd.service* + systemctl daemon-reload + systemctl reset-failed cri-dockerd.service + +- name: Cridockerd | Remove all cri-dockerd related files and binaries + command: | + rm -rf /etc/cri-dockerd + rm -f /usr/local/bin/cri-dockerd \ No newline at end of file diff --git a/builtin/core/roles/uninstall/cri/docker/tasks/docker.yaml b/builtin/core/roles/uninstall/cri/docker/tasks/docker.yaml new file mode 100644 index 000000000..a57dd97dd --- /dev/null +++ b/builtin/core/roles/uninstall/cri/docker/tasks/docker.yaml @@ -0,0 +1,38 @@ +--- +- name: Docker | Gracefully stop and disable the Docker service + ignore_errors: true + command: | + systemctl stop docker.service + systemctl disable docker.service + rm -rf /etc/systemd/system/docker.service* + systemctl daemon-reload + systemctl reset-failed docker.service + +- name: Docker | Completely uninstall containerd and remove all related files + block: + - name: Docker | Stop and disable the containerd service + ignore_errors: true + command: | + systemctl stop containerd.service + systemctl disable containerd.service + rm -rf /etc/systemd/system/containerd.service* + systemctl daemon-reload + systemctl reset-failed containerd.service + + - name: Docker | Remove all containerd-related files and binaries + command: | + rm -rf {{ .cri.containerd.data_root }} + rm -rf /etc/containerd + rm -rf /usr/local/bin/containerd* + rm -f /usr/local/bin/runc + rm -f /usr/local/bin/ctr + +- name: Docker | Remove all Docker residual files and binaries + command: | + rm -rf {{ .cri.docker.data_root }} + rm -rf /etc/docker + rm -rf /usr/local/bin/docker* + +- name: Docker | Remove the docker0 network interface + ignore_errors: true + command: ip link delete docker0 \ No newline at end of file diff --git a/builtin/core/roles/uninstall/cri/docker/tasks/main.yaml b/builtin/core/roles/uninstall/cri/docker/tasks/main.yaml new file mode 100644 index 000000000..9c607677a --- /dev/null +++ b/builtin/core/roles/uninstall/cri/docker/tasks/main.yaml @@ -0,0 +1,7 @@ +--- +- include_tasks: docker.yaml + +# uninstall cridockerd +- include_tasks: cridockerd.yaml + when: + - .cri.cridockerd_version | empty | not diff --git a/builtin/core/roles/uninstall/cri/meta/main.yaml b/builtin/core/roles/uninstall/cri/meta/main.yaml new file mode 100644 index 000000000..42f334943 --- /dev/null +++ b/builtin/core/roles/uninstall/cri/meta/main.yaml @@ -0,0 +1,10 @@ +--- +dependencies: + - role: uninstall/cri/containerd + when: .cri.container_manager | eq "containerd" + + - role: uninstall/cri/docker + when: .cri.container_manager | eq "docker" + + - role: uninstall/cri/crictl + diff --git a/builtin/core/roles/uninstall/etcd/tasks/main.yaml b/builtin/core/roles/uninstall/etcd/tasks/main.yaml new file mode 100644 index 000000000..b32dc3508 --- /dev/null +++ b/builtin/core/roles/uninstall/etcd/tasks/main.yaml @@ -0,0 +1,31 @@ +--- +- name: ETCD | Completely uninstall the etcd service and remove all related files + block: + - name: ETCD | Stop and disable the etcd systemd service + ignore_errors: true + command: | + systemctl stop etcd.service + systemctl disable etcd.service + rm -rf /etc/systemd/system/etcd.service* + systemctl daemon-reload + systemctl reset-failed etcd.service + - name: ETCD | Remove traffic priority rules for etcd ports + command: | + tc filter del dev eth0 parent 1: protocol ip prio 1 u32 match ip sport 2379 0xffff + tc filter del dev eth0 parent 1: protocol ip prio 1 u32 match ip sport 2380 0xffff + when: .etcd.traffic_priority + - name: ETCD | Delete all etcd data, configuration, and binaries + command: | + rm -rf {{ .etcd.env.data_dir }} + rm -rf /etc/ssl/etcd/ + rm -rf /etc/etcd.env + rm -rf /usr/local/bin/etcd* + +- name: ETCD | Uninstall backup-etcd timer and service, and remove backup scripts + ignore_errors: true + command: | + systemctl disable --now backup-etcd.timer + rm /etc/systemd/system/backup-etcd.timer + rm -rf /etc/systemd/system/backup-etcd.service* + rm /usr/local/bin/kube-scripts/backup_etcd.sh + systemctl daemon-reexec && systemctl daemon-reload diff --git a/builtin/core/roles/uninstall/image-registry/docker-compose/defaults/main.yaml b/builtin/core/roles/uninstall/image-registry/docker-compose/defaults/main.yaml new file mode 100644 index 000000000..42d27a1c7 --- /dev/null +++ b/builtin/core/roles/uninstall/image-registry/docker-compose/defaults/main.yaml @@ -0,0 +1,5 @@ +cri: + docker: + data_root: /var/lib/docker + containerd: + data_root: /var/lib/containerd \ No newline at end of file diff --git a/builtin/core/roles/uninstall/image-registry/docker-compose/tasks/main.yaml b/builtin/core/roles/uninstall/image-registry/docker-compose/tasks/main.yaml new file mode 100644 index 000000000..48c424c81 --- /dev/null +++ b/builtin/core/roles/uninstall/image-registry/docker-compose/tasks/main.yaml @@ -0,0 +1,38 @@ +--- +- name: DockerCompose | Gracefully stop and disable the Docker service + ignore_errors: true + command: | + systemctl stop docker.service + systemctl disable docker.service + rm -rf /etc/systemd/system/docker.service* + systemctl daemon-reload + systemctl reset-failed docker.service + +- name: DockerCompose | Completely uninstall containerd and remove all related files + block: + - name: DockerCompose | Stop and disable the containerd service + ignore_errors: true + command: | + systemctl stop containerd.service + systemctl disable containerd.service + rm -rf /etc/systemd/system/containerd.service* + systemctl daemon-reload + systemctl reset-failed containerd.service + + - name: DockerCompose | Remove all containerd-related files and binaries + command: | + rm -rf {{ .cri.containerd.data_root }} + rm -rf /etc/containerd + rm -rf /usr/local/bin/containerd* + rm -f /usr/local/bin/runc + rm -f /usr/local/bin/ctr + +- name: DockerCompose | Remove all Docker residual files and binaries + command: | + rm -rf {{ .cri.docker.data_root }} + rm -rf /etc/docker + rm -rf /usr/local/bin/docker* + +- name: DockerCompose | Remove the docker0 network interface + ignore_errors: true + command: ip link delete docker0 \ No newline at end of file diff --git a/builtin/core/roles/uninstall/image-registry/docker-registry/defaults/main.yaml b/builtin/core/roles/uninstall/image-registry/docker-registry/defaults/main.yaml new file mode 100644 index 000000000..fc25713fc --- /dev/null +++ b/builtin/core/roles/uninstall/image-registry/docker-registry/defaults/main.yaml @@ -0,0 +1,37 @@ +image_registry: + docker_registry: + version: 2 + config: + storage: nfs + nfs_dir: /share/registry + storage: + filesystem: + rootdir: /opt/docker-registry/data +# nfs_mount: /repository/registry # if set. will mount rootdirectory to nfs server in nfs_mount. +# azure: +# accountname: accountname +# accountkey: base64encodedaccountkey +# container: containername +# gcs: +# bucket: bucketname +# keyfile: /path/to/keyfile +# credentials: +# type: service_account +# project_id: project_id_string +# private_key_id: private_key_id_string +# private_key: private_key_string +# client_email: client@example.com +# client_id: client_id_string +# auth_uri: http://example.com/auth_uri +# token_uri: http://example.com/token_uri +# auth_provider_x509_cert_url: http://example.com/provider_cert_url +# client_x509_cert_url: http://example.com/client_cert_url +# rootdirectory: /gcs/object/name/prefix +# s3: +# accesskey: awsaccesskey +# secretkey: awssecretkey +# region: us-west-1 +# regionendpoint: http://myobjects.local +# bucket: bucketname +# keyid: mykeyid +# rootdirectory: /s3/object/name/prefix diff --git a/builtin/core/roles/uninstall/image-registry/docker-registry/tasks/main.yaml b/builtin/core/roles/uninstall/image-registry/docker-registry/tasks/main.yaml new file mode 100644 index 000000000..669b96156 --- /dev/null +++ b/builtin/core/roles/uninstall/image-registry/docker-registry/tasks/main.yaml @@ -0,0 +1,19 @@ +- name: DockerRegistry | Gracefully stop and disable the Docker Registry service + ignore_errors: true + command: | + systemctl stop docker-registry.service + systemctl disable docker-registry.service + rm -rf /etc/systemd/system/docker-registry.service* + systemctl daemon-reload + systemctl reset-failed docker-registry.service + +- name: DockerRegistry | Unmount NFS storage for Docker Registry if configured + when: + - .image_registry.docker_registry.storage.filesystem.nfs_mount | empty | not + - .groups.nfs | default list | len | eq 1 + command: | + unmount {{ .image_registry.docker_registry.storage.filesystem.rootdir }} + +- name: DockerRegistry | Remove all residual Docker Registry files and directories + command: | + rm -rf /opt/docker-registry/ \ No newline at end of file diff --git a/builtin/core/roles/uninstall/image-registry/harbor/defaults/main.yaml b/builtin/core/roles/uninstall/image-registry/harbor/defaults/main.yaml new file mode 100644 index 000000000..4f1fd9f7c --- /dev/null +++ b/builtin/core/roles/uninstall/image-registry/harbor/defaults/main.yaml @@ -0,0 +1,3 @@ +image_registry: + harbor: + data_dir: /opt/harbor/data diff --git a/builtin/core/roles/uninstall/image-registry/harbor/tasks/main.yaml b/builtin/core/roles/uninstall/image-registry/harbor/tasks/main.yaml new file mode 100644 index 000000000..dcecf07b8 --- /dev/null +++ b/builtin/core/roles/uninstall/image-registry/harbor/tasks/main.yaml @@ -0,0 +1,12 @@ +- name: Harbor | Gracefully stop and disable the Harbor service + ignore_errors: true + command: | + systemctl stop harbor.service + systemctl disable harbor.service + rm -rf /etc/systemd/system/harbor.service* + systemctl daemon-reload + systemctl reset-failed harbor.service + +- name: Harbor | Remove all residual Harbor files and directories + command: | + rm -rf /opt/harbor/ \ No newline at end of file diff --git a/builtin/core/roles/uninstall/image-registry/keepalived/tasks/main.yaml b/builtin/core/roles/uninstall/image-registry/keepalived/tasks/main.yaml new file mode 100644 index 000000000..06d51eaa9 --- /dev/null +++ b/builtin/core/roles/uninstall/image-registry/keepalived/tasks/main.yaml @@ -0,0 +1,8 @@ +- name: Keepalived | Remove ARP and IP address entries associated with kube-vip + command: | + ip neigh show | grep {{ .image_registry.ha_vip }} | awk '{print $1 " dev " $3}' | xargs -r -L1 ip neigh delete + ip -o addr show | grep {{ .image_registry.ha_vip }} | awk '{system("ip addr del "$4" dev "$2)}' + +- name: Keepalived | Remove all residual Keepalived files and directories + command: | + rm -rf /opt/keepalived/ \ No newline at end of file diff --git a/builtin/core/roles/uninstall/image-registry/meta/main.yaml b/builtin/core/roles/uninstall/image-registry/meta/main.yaml new file mode 100644 index 000000000..748dfa37f --- /dev/null +++ b/builtin/core/roles/uninstall/image-registry/meta/main.yaml @@ -0,0 +1,14 @@ +dependencies: + - role: uninstall/image-registry/harbor + when: .image_registry.type | eq "harbor" + + - role: uninstall/image-registry/docker-registry + when: .image_registry.type | eq "docker-registry" + + - role: uninstall/image-registry/keepalived + when: + - .image_registry.ha_vip | empty | not + - .groups.image_registry | len | lt 1 + + - role: uninstall/image-registry/docker-compose + when: .deleteCRI \ No newline at end of file diff --git a/builtin/core/roles/uninstall/kubernetes/tasks/kubernetes.yaml b/builtin/core/roles/uninstall/kubernetes/tasks/kubernetes.yaml new file mode 100644 index 000000000..083555fbe --- /dev/null +++ b/builtin/core/roles/uninstall/kubernetes/tasks/kubernetes.yaml @@ -0,0 +1,26 @@ +--- +- name: Kubernetes | Completely reset the node using kubeadm + ignore_errors: true + command: | + kubeadm reset -f + +- name: Kubernetes | Gracefully stop and disable the kubelet service + ignore_errors: true + command: | + systemctl stop kubelet.service + systemctl disable kubelet.service + rm -rf /etc/systemd/system/kubelet.service* + systemctl daemon-reload + systemctl reset-failed kubelet.service + +- name: Kubernetes | Remove all residual Kubernetes files and directories + command: | + rm -rf /usr/local/bin/kubeadm + rm -rf /usr/local/bin/kubelet + rm -rf /usr/local/bin/kubectl + rm -rf /var/lib/kubelet/ + # If /var/log/pods/ is not cleaned up, static pods may accumulate unexpected restarts due to lingering log files interfering with their lifecycle. + rm -rf /var/log/pods/ + rm -rf /etc/kubernetes/ + rm -rf .kube/config + rm -rf /var/lib/etcd \ No newline at end of file diff --git a/builtin/core/roles/uninstall/kubernetes/tasks/main.yaml b/builtin/core/roles/uninstall/kubernetes/tasks/main.yaml new file mode 100644 index 000000000..a6d86c930 --- /dev/null +++ b/builtin/core/roles/uninstall/kubernetes/tasks/main.yaml @@ -0,0 +1,8 @@ +--- +- include_tasks: kubernetes.yaml + +- include_tasks: network.yaml + +- name: Kubernetes | Remove all residual Kubekey files and directories + command: | + rm -rf /etc/kubekey/ \ No newline at end of file diff --git a/builtin/core/roles/uninstall/kubernetes/tasks/network.yaml b/builtin/core/roles/uninstall/kubernetes/tasks/network.yaml new file mode 100644 index 000000000..81b896997 --- /dev/null +++ b/builtin/core/roles/uninstall/kubernetes/tasks/network.yaml @@ -0,0 +1,51 @@ +--- +- name: Network | Thoroughly clean up iptables and network interfaces + ignore_errors: true + command: | + iptables -F + iptables -X + iptables -F -t nat + iptables -X -t nat + ipvsadm -C + ip link del kube-ipvs0 + ip link del nodelocaldns + ip link del cni0 + {{- if .cni.type | eq "flannel" }} + ip link del flannel.1 + ip link del flannel-v6.1 + ip link del flannel-wg + ip link del flannel-wg-v6 + {{- end }} + {{- if .cni.type | eq "cilium" }} + ip link del cilium_host + ip link del cilium_vxlan + {{- end }} + {{- if .cni.type | eq "calico" }} + ip -br link show | grep cali[a-f0-9]* | awk -F @ '{print $1}' | xargs -r -t -n 1 ip link del + {{- end }} + ip netns show 2>/dev/null | grep cni- | awk '{print $1}' | xargs -r -t -n 1 ip netns del + +- name: Network | Remove all CNI network configuration and state + command: | + rm -rf /etc/cni/net.d/ + rm -rf /var/lib/cni/ + {{- if .cni.type | eq "calico" }} + rm -rf /usr/local/bin/calicoctl + {{- end }} + +- name: Network | Remove ARP and IP address entries for kube-vip + when: eq .kubernetes.control_plane_endpoint.type "kube_vip" + command: | + ip neigh show | grep {{ .kubernetes.control_plane_endpoint.kube_vip.address }} | awk '{print $1 " dev " $3}' | xargs -r -L1 ip neigh delete + ip -o addr show | grep {{ .kubernetes.control_plane_endpoint.kube_vip.address }} | awk '{system("ip addr del "$4" dev "$2)}' + +- name: Network | Restart container runtime to rebuild iptables rules + ignore_errors: true + when: or (.deleteCRI | not) (.groups.image_registry | default list | has .inventory_hostname) + command: | + {{- if .cri.container_manager | eq "docker" }} + systemctl restart containerd + systemctl restart docker + {{- else if .cri.container_manager | eq "containerd" }} + systemctl restart containerd + {{- end }} \ No newline at end of file diff --git a/cmd/controller-manager/app/options/common.go b/cmd/controller-manager/app/options/common.go new file mode 100644 index 000000000..d68129f56 --- /dev/null +++ b/cmd/controller-manager/app/options/common.go @@ -0,0 +1,164 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "context" + "flag" + "fmt" + "os" + "runtime" + "runtime/pprof" + "strings" + + "github.com/cockroachdb/errors" + "github.com/google/gops/agent" + "github.com/spf13/pflag" + "k8s.io/klog/v2" +) + +// ====================================================================================== +// PROFILING +// ====================================================================================== + +var ( + profileName string + profileOutput string +) + +// AddProfilingFlags to NewControllerManagerCommand +func AddProfilingFlags(flags *pflag.FlagSet) { + flags.StringVar(&profileName, "profile", "none", "Name of profile to capture. One of (none|cpu|heap|goroutine|threadcreate|block|mutex)") + flags.StringVar(&profileOutput, "profile-output", "profile.pprof", "Name of the file to write the profile to") +} + +// InitProfiling for profileName +func InitProfiling(ctx context.Context) error { + var ( + f *os.File + err error + ) + + switch profileName { + case "none": + return nil + case "cpu": + f, err = os.Create(profileOutput) + if err != nil { + return errors.Wrap(err, "failed to create cpu profile") + } + + err = pprof.StartCPUProfile(f) + if err != nil { + return errors.Wrap(err, "failed to start cpu profile") + } + // Block and mutex profiles need a call to Set{Block,Mutex}ProfileRate to + // output anything. We choose to sample all events. + case "block": + runtime.SetBlockProfileRate(1) + case "mutex": + runtime.SetMutexProfileFraction(1) + default: + // Check the profile name is valid. + if profile := pprof.Lookup(profileName); profile == nil { + return errors.Errorf("unknown profile '%s'", profileName) + } + } + + // If the command is interrupted before the end (ctrl-c), flush the + // profiling files + go func() { + <-ctx.Done() + if err := f.Close(); err != nil { + fmt.Printf("failed to close file. file: %v. error: %v \n", profileOutput, err) + } + if err := FlushProfiling(); err != nil { + fmt.Printf("failed to FlushProfiling. file: %v. error: %v \n", profileOutput, err) + } + }() + + return nil +} + +// FlushProfiling to local file +func FlushProfiling() error { + switch profileName { + case "none": + return nil + case "cpu": + pprof.StopCPUProfile() + case "heap": + runtime.GC() + + fallthrough + default: + profile := pprof.Lookup(profileName) + if profile == nil { + return nil + } + + f, err := os.Create(profileOutput) + if err != nil { + return errors.Wrapf(err, "failed to create profile %s", profileName) + } + defer f.Close() + + if err := profile.WriteTo(f, 0); err != nil { + return errors.Wrapf(err, "failed to write profile %s", profileName) + } + } + + return nil +} + +// ====================================================================================== +// GOPS +// ====================================================================================== + +var gops bool + +// AddGOPSFlags to NewControllerManagerCommand +func AddGOPSFlags(flags *pflag.FlagSet) { + flags.BoolVar(&gops, "gops", false, "Whether to enable gops or not. When enabled this option, controller-manager will listen on a random port on 127.0.0.1, then you can use the gops tool to list and diagnose the controller-manager currently running.") +} + +// InitGOPS if gops is true +func InitGOPS() error { + if gops { + // Add agent to report additional information such as the current stack trace, Go version, memory stats, etc. + // Bind to a random port on address 127.0.0.1 + if err := agent.Listen(agent.Options{}); err != nil { + return errors.Wrap(err, "failed to listen gops agent") + } + } + + return nil +} + +// ====================================================================================== +// KLOG +// ====================================================================================== + +// AddKlogFlags to NewControllerManagerCommand +func AddKlogFlags(fs *pflag.FlagSet) { + local := flag.NewFlagSet("klog", flag.ExitOnError) + klog.InitFlags(local) + local.VisitAll(func(fl *flag.Flag) { + fl.Name = strings.Replace(fl.Name, "_", "-", -1) + fs.AddGoFlag(fl) + }) +} diff --git a/cmd/controller-manager/app/options/controller_manager.go b/cmd/controller-manager/app/options/controller_manager.go new file mode 100644 index 000000000..a8499f5de --- /dev/null +++ b/cmd/controller-manager/app/options/controller_manager.go @@ -0,0 +1,146 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "fmt" + "sort" + "strings" + + "github.com/cockroachdb/errors" + cliflag "k8s.io/component-base/cli/flag" + ctrl "sigs.k8s.io/controller-runtime" +) + +// ControllerManagerServerOptions for NewControllerManagerServerOptions +type ControllerManagerServerOptions struct { + // Debug mode, after a successful execution of Playbook, will retain runtime data, which includes task execution status and parameters. + Debug bool + + MaxConcurrentReconciles int + LeaderElection bool + LeaderElectionID string + // LeaderElectionResourceLock determines which resource lock to use for leader election, + // defaults to "leases". Change this value only if you know what you are doing. + // + // If you are using `configmaps`/`endpoints` resource lock and want to migrate to "leases", + // you might do so by migrating to the respective multilock first ("configmapsleases" or "endpointsleases"), + // which will acquire a leader lock on both resources. + // After all your users have migrated to the multilock, you can go ahead and migrate to "leases". + // Please also keep in mind, that users might skip versions of your controller. + // + // Note: before controller-runtime version v0.7, it was set to "configmaps". + // And from v0.7 to v0.11, the default was "configmapsleases", which was + // used to migrate from configmaps to leases. + // Since the default was "configmapsleases" for over a year, spanning five minor releases, + // any actively maintained operators are very likely to have a released version that uses + // "configmapsleases". Therefore defaulting to "leases" should be safe since v0.12. + // + // So, what do you have to do when you are updating your controller-runtime dependency + // from a lower version to v0.12 or newer? + // - If your operator matches at least one of these conditions: + // - the LeaderElectionResourceLock in your operator has already been explicitly set to "leases" + // - the old controller-runtime version is between v0.7.0 and v0.11.x and the + // LeaderElectionResourceLock wasn't set or was set to "leases"/"configmapsleases"/"endpointsleases" + // feel free to update controller-runtime to v0.12 or newer. + // - Otherwise, you may have to take these steps: + // 1. update controller-runtime to v0.12 or newer in your go.mod + // 2. set LeaderElectionResourceLock to "configmapsleases" (or "endpointsleases") + // 3. package your operator and upgrade it in all your clusters + // 4. only if you have finished 3, you can remove the LeaderElectionResourceLock to use the default "leases" + // Otherwise, your operator might end up with multiple running instances that + // each acquired leadership through different resource locks during upgrades and thus + // act on the same resources concurrently. + LeaderElectionResourceLock string + // ControllerGates is the list of controller gates to enable or disable controller. + // '*' means "all enabled by default controllers" + // 'foo' means "enable 'foo'" + // '-foo' means "disable 'foo'" + // first item for a particular name wins. + // e.g. '-foo,foo' means "disable foo", 'foo,-foo' means "enable foo" + // * has the lowest priority. + // e.g. *,-foo, means "disable 'foo'" + ControllerGates []string + Controllers []Controller +} + +// Keys returns a sorted list of controller names +func (o *ControllerManagerServerOptions) controllerKeys() string { + var keys = make([]string, 0) + for _, c := range o.Controllers { + keys = append(keys, c.Name()) + } + sort.Strings(keys) + + return strings.Join(keys, ",") +} + +// NewControllerManagerServerOptions for NewControllerManagerCommand +func NewControllerManagerServerOptions() *ControllerManagerServerOptions { + return &ControllerManagerServerOptions{ + MaxConcurrentReconciles: 1, + Controllers: controllers, + } +} + +// Flags add to NewControllerManagerCommand +func (o *ControllerManagerServerOptions) Flags() cliflag.NamedFlagSets { + fss := cliflag.NamedFlagSets{} + gfs := fss.FlagSet("generic") + gfs.BoolVar(&o.Debug, "debug", o.Debug, "Debug mode, after a successful execution of Playbook, "+"will retain runtime data, which includes task execution status and parameters.") + cfs := fss.FlagSet("controller-manager") + cfs.IntVar(&o.MaxConcurrentReconciles, "max-concurrent-reconciles", o.MaxConcurrentReconciles, "The number of maximum concurrent reconciles for controller.") + cfs.BoolVar(&o.LeaderElection, "leader-election", o.LeaderElection, "Whether to enable leader election for controller-manager.") + cfs.StringVar(&o.LeaderElectionID, "leader-election-id", o.LeaderElectionID, "Whether to enable leader election for controller-manager.") + cfs.StringVar(&o.LeaderElectionResourceLock, "leader-election-lock", o.LeaderElectionResourceLock, " which resource lock to use for leader election.") + cfs.StringSliceVar(&o.ControllerGates, "controllers", []string{"*"}, fmt.Sprintf(""+ + "A list of controllers to enable. '*' enables all on-by-default controllers, 'foo' enables the controller "+ + "named 'foo', '-foo' disables the controller named 'foo'.\nAll controllers: %s", + o.controllerKeys())) + + return fss +} + +// Complete for ControllerManagerServerOptions +func (o *ControllerManagerServerOptions) Complete() { + // do nothing + if o.MaxConcurrentReconciles == 0 { + o.MaxConcurrentReconciles = 1 + } +} + +var controllers []Controller + +// Register controller +func Register(reconciler Controller) error { + for _, c := range controllers { + if c.Name() == reconciler.Name() { + return errors.Errorf("%s has register", reconciler.Name()) + } + } + controllers = append(controllers, reconciler) + + return nil +} + +// Controller should add in ctrl.manager +type Controller interface { + // setup reconcile with manager + SetupWithManager(mgr ctrl.Manager, o ControllerManagerServerOptions) error + // the Name of controller + Name() string +} diff --git a/cmd/controller-manager/app/server.go b/cmd/controller-manager/app/server.go new file mode 100644 index 000000000..e09467c2c --- /dev/null +++ b/cmd/controller-manager/app/server.go @@ -0,0 +1,72 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "context" + + "github.com/spf13/cobra" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" + + "github.com/kubesphere/kubekey/v4/cmd/controller-manager/app/options" + "github.com/kubesphere/kubekey/v4/pkg/manager" +) + +// NewControllerManagerCommand operator command. +func NewControllerManagerCommand() *cobra.Command { + o := options.NewControllerManagerServerOptions() + ctx := signals.SetupSignalHandler() + + cmd := &cobra.Command{ + Use: "controller-manager", + Short: "kubekey controller manager", + PersistentPreRunE: func(*cobra.Command, []string) error { + if err := options.InitGOPS(); err != nil { + return err + } + + return options.InitProfiling(ctx) + }, + PersistentPostRunE: func(*cobra.Command, []string) error { + return options.FlushProfiling() + }, + RunE: func(*cobra.Command, []string) error { + o.Complete() + + return run(ctx, o) + }, + } + + // add common flag + flags := cmd.PersistentFlags() + options.AddProfilingFlags(flags) + options.AddKlogFlags(flags) + options.AddGOPSFlags(flags) + + fs := cmd.Flags() + for _, f := range o.Flags().FlagSets { + fs.AddFlagSet(f) + } + + cmd.AddCommand(newVersionCommand()) + + return cmd +} + +func run(ctx context.Context, o *options.ControllerManagerServerOptions) error { + return manager.NewControllerManager(o).Run(ctx) +} diff --git a/cmd/controller-manager/app/version.go b/cmd/controller-manager/app/version.go new file mode 100644 index 000000000..84a80c450 --- /dev/null +++ b/cmd/controller-manager/app/version.go @@ -0,0 +1,33 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "github.com/spf13/cobra" + + "github.com/kubesphere/kubekey/v4/version" +) + +func newVersionCommand() *cobra.Command { + return &cobra.Command{ + Use: "version", + Short: "Print the version of KubeSphere controller-manager", + Run: func(cmd *cobra.Command, _ []string) { + cmd.Println(version.Get()) + }, + } +} diff --git a/cmd/controller-manager/controller_manager.go b/cmd/controller-manager/controller_manager.go new file mode 100644 index 000000000..bdcb00475 --- /dev/null +++ b/cmd/controller-manager/controller_manager.go @@ -0,0 +1,33 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "os" + + "github.com/kubesphere/kubekey/v4/cmd/controller-manager/app" + _ "github.com/kubesphere/kubekey/v4/pkg/controllers/core" + _ "github.com/kubesphere/kubekey/v4/pkg/controllers/infrastructure" +) + +func main() { + if err := app.NewControllerManagerCommand().Execute(); err != nil { + fmt.Printf("%+v", err) + os.Exit(1) + } +} diff --git a/cmd/kk/apis/kubekey/v1alpha2/addons_types.go b/cmd/kk/apis/kubekey/v1alpha2/addons_types.go deleted file mode 100644 index 5500317b1..000000000 --- a/cmd/kk/apis/kubekey/v1alpha2/addons_types.go +++ /dev/null @@ -1,44 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1alpha2 - -type Addon struct { - Name string `yaml:"name" json:"name,omitempty"` - Namespace string `yaml:"namespace" json:"namespace,omitempty"` - Sources Sources `yaml:"sources" json:"sources,omitempty"` - Retries int `yaml:"retries" json:"retries,omitempty"` - Delay int `yaml:"delay" json:"delay,omitempty"` -} - -type Sources struct { - Chart Chart `yaml:"chart" json:"chart,omitempty"` - Yaml Yaml `yaml:"yaml" json:"yaml,omitempty"` -} - -type Chart struct { - Name string `yaml:"name" json:"name,omitempty"` - Repo string `yaml:"repo" json:"repo,omitempty"` - Path string `yaml:"path" json:"path,omitempty"` - Version string `yaml:"version" json:"version,omitempty"` - ValuesFile string `yaml:"valuesFile" json:"valuesFile,omitempty"` - Values []string `yaml:"values" json:"values,omitempty"` - Wait bool `yaml:"wait" json:"wait,omitempty"` -} - -type Yaml struct { - Path []string `yaml:"path" json:"path,omitempty"` -} diff --git a/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go b/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go deleted file mode 100644 index 1bd4f7799..000000000 --- a/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go +++ /dev/null @@ -1,305 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1alpha2 - -import ( - "fmt" - "regexp" - "strconv" - "strings" - - "github.com/pkg/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -// ClusterSpec defines the desired state of Cluster -type ClusterSpec struct { - Hosts []HostCfg `yaml:"hosts" json:"hosts,omitempty"` - RoleGroups map[string][]string `yaml:"roleGroups" json:"roleGroups,omitempty"` - ControlPlaneEndpoint ControlPlaneEndpoint `yaml:"controlPlaneEndpoint" json:"controlPlaneEndpoint,omitempty"` - System System `yaml:"system" json:"system,omitempty"` - Etcd EtcdCluster `yaml:"etcd" json:"etcd,omitempty"` - DNS DNS `yaml:"dns" json:"dns,omitempty"` - Kubernetes Kubernetes `yaml:"kubernetes" json:"kubernetes,omitempty"` - Network NetworkConfig `yaml:"network" json:"network,omitempty"` - Storage StorageConfig `yaml:"storage" json:"storage,omitempty"` - Registry RegistryConfig `yaml:"registry" json:"registry,omitempty"` - Addons []Addon `yaml:"addons" json:"addons,omitempty"` - KubeSphere KubeSphere `json:"kubesphere,omitempty"` -} - -type Cluster struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec ClusterSpec `json:"spec,omitempty"` -} - -// HostCfg defines host information for cluster. -type HostCfg struct { - Name string `yaml:"name,omitempty" json:"name,omitempty"` - Address string `yaml:"address,omitempty" json:"address,omitempty"` - InternalAddress string `yaml:"internalAddress,omitempty" json:"internalAddress,omitempty"` - Port int `yaml:"port,omitempty" json:"port,omitempty"` - User string `yaml:"user,omitempty" json:"user,omitempty"` - Password string `yaml:"password,omitempty" json:"password,omitempty"` - PrivateKey string `yaml:"privateKey,omitempty" json:"privateKey,omitempty"` - PrivateKeyPath string `yaml:"privateKeyPath,omitempty" json:"privateKeyPath,omitempty"` - Arch string `yaml:"arch,omitempty" json:"arch,omitempty"` - Timeout *int64 `yaml:"timeout,omitempty" json:"timeout,omitempty"` - - // Labels defines the kubernetes labels for the node. - Labels map[string]string `yaml:"labels,omitempty" json:"labels,omitempty"` -} - -// ControlPlaneEndpoint defines the control plane endpoint information for cluster. -type ControlPlaneEndpoint struct { - InternalLoadbalancer string `yaml:"internalLoadbalancer" json:"internalLoadbalancer,omitempty"` - Domain string `yaml:"domain" json:"domain,omitempty"` - ExternalDNS *bool `yaml:"externalDNS" json:"externalDNS"` - Address string `yaml:"address" json:"address,omitempty"` - Port int `yaml:"port" json:"port,omitempty"` - KubeVip KubeVip `yaml:"kubevip" json:"kubevip,omitempty"` -} - -type KubeVip struct { - Mode string `yaml:"mode" json:"mode,omitempty"` -} - -// CustomScripts defines the custom shell scripts for each node to exec before and finished kubernetes install. -type CustomScripts struct { - Name string `yaml:"name" json:"name,omitempty"` - Bash string `yaml:"bash" json:"bash,omitempty"` - Materials []string `yaml:"materials" json:"materials,omitempty"` -} - -// System defines the system config for each node in cluster. -type System struct { - NtpServers []string `yaml:"ntpServers" json:"ntpServers,omitempty"` - Timezone string `yaml:"timezone" json:"timezone,omitempty"` - Rpms []string `yaml:"rpms" json:"rpms,omitempty"` - Debs []string `yaml:"debs" json:"debs,omitempty"` - PreInstall []CustomScripts `yaml:"preInstall" json:"preInstall,omitempty"` - PostInstall []CustomScripts `yaml:"postInstall" json:"postInstall,omitempty"` - SkipConfigureOS bool `yaml:"skipConfigureOS" json:"skipConfigureOS,omitempty"` -} - -// RegistryConfig defines the configuration information of the image's repository. -type RegistryConfig struct { - Type string `yaml:"type" json:"type,omitempty"` - RegistryMirrors []string `yaml:"registryMirrors" json:"registryMirrors,omitempty"` - InsecureRegistries []string `yaml:"insecureRegistries" json:"insecureRegistries,omitempty"` - PrivateRegistry string `yaml:"privateRegistry" json:"privateRegistry,omitempty"` - DataRoot string `yaml:"dataRoot" json:"dataRoot,omitempty"` - NamespaceOverride string `yaml:"namespaceOverride" json:"namespaceOverride,omitempty"` - BridgeIP string `yaml:"bridgeIP" json:"bridgeIP,omitempty"` - Auths runtime.RawExtension `yaml:"auths" json:"auths,omitempty"` -} - -// KubeSphere defines the configuration information of the KubeSphere. -type KubeSphere struct { - Enabled bool `json:"enabled,omitempty"` - Version string `json:"version,omitempty"` - Configurations string `json:"configurations,omitempty"` -} - -// GenerateCertSANs is used to generate cert sans for cluster. -func (cfg *ClusterSpec) GenerateCertSANs() []string { - clusterSvc := fmt.Sprintf("kubernetes.default.svc.%s", cfg.Kubernetes.DNSDomain) - defaultCertSANs := []string{"kubernetes", "kubernetes.default", "kubernetes.default.svc", clusterSvc, "localhost", "127.0.0.1"} - extraCertSANs := make([]string, 0) - - extraCertSANs = append(extraCertSANs, cfg.ControlPlaneEndpoint.Domain) - - if cfg.ControlPlaneEndpoint.Address != "" { - extraCertSANs = append(extraCertSANs, cfg.ControlPlaneEndpoint.Address) - } - - for _, host := range cfg.Hosts { - extraCertSANs = append(extraCertSANs, host.Name) - extraCertSANs = append(extraCertSANs, fmt.Sprintf("%s.%s", host.Name, cfg.Kubernetes.DNSDomain)) - if host.Address != cfg.ControlPlaneEndpoint.Address { - extraCertSANs = append(extraCertSANs, host.Address) - } - if host.InternalAddress != host.Address && host.InternalAddress != cfg.ControlPlaneEndpoint.Address { - extraCertSANs = append(extraCertSANs, host.InternalAddress) - } - } - - extraCertSANs = append(extraCertSANs, util.ParseIp(cfg.Network.KubeServiceCIDR)[0]) - - defaultCertSANs = append(defaultCertSANs, extraCertSANs...) - - if cfg.Kubernetes.ApiserverCertExtraSans != nil { - defaultCertSANs = append(defaultCertSANs, cfg.Kubernetes.ApiserverCertExtraSans...) - } - - return defaultCertSANs -} - -// GroupHosts is used to group hosts according to the configuration file.s -func (cfg *ClusterSpec) GroupHosts() map[string][]*KubeHost { - hostMap := make(map[string]*KubeHost) - for _, hostCfg := range cfg.Hosts { - host := toHosts(hostCfg) - hostMap[host.Name] = host - } - - roleGroups := cfg.ParseRolesList(hostMap) - - //Check that the parameters under roleGroups are incorrect - if len(roleGroups[Master]) == 0 && len(roleGroups[ControlPlane]) == 0 { - logger.Log.Fatal(errors.New("The number of master/control-plane cannot be 0")) - } - if len(roleGroups[Etcd]) == 0 && cfg.Etcd.Type == KubeKey { - logger.Log.Fatal(errors.New("The number of etcd cannot be 0")) - } - - for _, host := range roleGroups[ControlPlane] { - host.SetRole(Master) - roleGroups[Master] = append(roleGroups[Master], host) - } - - return roleGroups -} - -// +kubebuilder:object:generate=false -type KubeHost struct { - *connector.BaseHost - Labels map[string]string -} - -func toHosts(cfg HostCfg) *KubeHost { - host := connector.NewHost() - host.Name = cfg.Name - host.Address = cfg.Address - host.InternalAddress = cfg.InternalAddress - host.Port = cfg.Port - host.User = cfg.User - host.Password = cfg.Password - host.PrivateKey = cfg.PrivateKey - host.PrivateKeyPath = cfg.PrivateKeyPath - host.Arch = cfg.Arch - host.Timeout = *cfg.Timeout - - kubeHost := &KubeHost{ - BaseHost: host, - Labels: cfg.Labels, - } - return kubeHost -} - -// ClusterIP is used to get the kube-apiserver service address inside the cluster. -func (cfg *ClusterSpec) ClusterIP() string { - return util.ParseIp(cfg.Network.KubeServiceCIDR)[0] -} - -// CorednsClusterIP is used to get the coredns service address inside the cluster. -func (cfg *ClusterSpec) CorednsClusterIP() string { - return util.ParseIp(cfg.Network.KubeServiceCIDR)[2] -} - -// ClusterDNS is used to get the dns server address inside the cluster. -func (cfg *ClusterSpec) ClusterDNS() string { - if cfg.Kubernetes.EnableNodelocaldns() { - return "169.254.25.10" - } else { - return cfg.CorednsClusterIP() - } -} - -// ParseRolesList is used to parse the host grouping list. -func (cfg *ClusterSpec) ParseRolesList(hostMap map[string]*KubeHost) map[string][]*KubeHost { - roleGroupLists := make(map[string][]*KubeHost) - for role, hosts := range cfg.RoleGroups { - roleGroup := make([]string, 0) - for _, host := range hosts { - h := make([]string, 0) - if strings.Contains(host, "[") && strings.Contains(host, "]") && strings.Contains(host, ":") { - rangeHosts := getHostsRange(host, hostMap, role) - h = append(h, rangeHosts...) - } else { - if err := hostVerify(hostMap, host, role); err != nil { - logger.Log.Fatal(err) - } - h = append(h, host) - } - - roleGroup = append(roleGroup, h...) - for _, hostName := range h { - if h, ok := hostMap[hostName]; ok { - roleGroupAppend(roleGroupLists, role, h) - } else { - logger.Log.Fatal(fmt.Errorf("incorrect nodeName under roleGroups/%s in the configuration file", role)) - } - } - } - } - - return roleGroupLists -} - -func roleGroupAppend(roleGroupLists map[string][]*KubeHost, role string, host *KubeHost) { - host.SetRole(role) - r := roleGroupLists[role] - r = append(r, host) - roleGroupLists[role] = r -} - -func getHostsRange(rangeStr string, hostMap map[string]*KubeHost, group string) []string { - hostRangeList := make([]string, 0) - r := regexp.MustCompile(`\[(\d+)\:(\d+)\]`) - nameSuffix := r.FindStringSubmatch(rangeStr) - namePrefix := strings.Split(rangeStr, nameSuffix[0])[0] - nameSuffixStart, _ := strconv.Atoi(nameSuffix[1]) - nameSuffixEnd, _ := strconv.Atoi(nameSuffix[2]) - for i := nameSuffixStart; i <= nameSuffixEnd; i++ { - if err := hostVerify(hostMap, fmt.Sprintf("%s%d", namePrefix, i), group); err != nil { - logger.Log.Fatal(err) - } - hostRangeList = append(hostRangeList, fmt.Sprintf("%s%d", namePrefix, i)) - } - return hostRangeList -} - -func hostVerify(hostMap map[string]*KubeHost, hostName string, group string) error { - if _, ok := hostMap[hostName]; !ok { - return fmt.Errorf("[%s] is in [%s] group, but not in hosts list", hostName, group) - } - return nil -} - -func (c ControlPlaneEndpoint) IsInternalLBEnabled() bool { - return c.InternalLoadbalancer == Haproxy -} - -func (c ControlPlaneEndpoint) IsInternalLBEnabledVip() bool { - return c.InternalLoadbalancer == Kubevip -} - -// EnableExternalDNS is used to determine whether to use external dns to resolve kube-apiserver domain. -func (c *ControlPlaneEndpoint) EnableExternalDNS() bool { - if c.ExternalDNS == nil { - return false - } - return *c.ExternalDNS -} diff --git a/cmd/kk/apis/kubekey/v1alpha2/default.go b/cmd/kk/apis/kubekey/v1alpha2/default.go deleted file mode 100644 index 64701675f..000000000 --- a/cmd/kk/apis/kubekey/v1alpha2/default.go +++ /dev/null @@ -1,352 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1alpha2 - -import ( - "fmt" - "os" - "strings" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -const ( - DefaultPreDir = "kubekey" - DefaultTmpDir = "/tmp/kubekey" - DefaultSSHPort = 22 - DefaultLBPort = 6443 - DefaultApiserverPort = 6443 - DefaultLBDomain = "lb.kubesphere.local" - DefaultNetworkPlugin = "calico" - DefaultPodsCIDR = "10.233.64.0/18" - DefaultServiceCIDR = "10.233.0.0/18" - DefaultKubeImageNamespace = "kubesphere" - DefaultClusterName = "cluster.local" - DefaultDNSDomain = "cluster.local" - DefaultArch = "amd64" - DefaultSSHTimeout = 30 - DefaultEtcdVersion = "v3.5.6" - DefaultEtcdPort = "2379" - DefaultDockerVersion = "24.0.6" - DefaultContainerdVersion = "1.7.8" - DefaultRuncVersion = "v1.1.10" - DefaultCrictlVersion = "v1.24.0" - DefaultKubeVersion = "v1.23.10" - DefaultCalicoVersion = "v3.26.1" - DefaultFlannelVersion = "v0.21.3" - DefaultFlannelCniPluginVersion = "v1.1.2" - DefaultCniVersion = "v1.2.0" - DefaultCiliumVersion = "v1.11.7" - DefaulthybridnetVersion = "v0.8.6" - DefaultKubeovnVersion = "v1.10.6" - DefalutMultusVersion = "v3.8" - DefaultHelmVersion = "v3.9.0" - DefaultDockerComposeVersion = "v2.2.2" - DefaultRegistryVersion = "2" - DefaultHarborVersion = "v2.5.3" - DefaultMaxPods = 110 - DefaultPodPidsLimit = 10000 - DefaultNodeCidrMaskSize = 24 - DefaultIPIPMode = "Always" - DefaultVXLANMode = "Never" - DefaultVethMTU = 0 - DefaultBackendMode = "vxlan" - DefaultProxyMode = "ipvs" - DefaultCrioEndpoint = "unix:///var/run/crio/crio.sock" - DefaultContainerdEndpoint = "unix:///run/containerd/containerd.sock" - DefaultIsulaEndpoint = "unix:///var/run/isulad.sock" - Etcd = "etcd" - Master = "master" - ControlPlane = "control-plane" - Worker = "worker" - K8s = "k8s" - Registry = "registry" - DefaultEtcdBackupDir = "/var/backups/kube_etcd" - DefaultEtcdBackupPeriod = 1440 - DefaultKeepBackNumber = 5 - DefaultEtcdBackupScriptDir = "/usr/local/bin/kube-scripts" - DefaultPodGateway = "10.233.64.1" - DefaultJoinCIDR = "100.64.0.0/16" - DefaultNetworkType = "geneve" - DefaultTunnelType = "geneve" - DefaultPodNicType = "veth-pair" - DefaultModules = "kube_ovn_fastpath.ko" - DefaultRPMs = "openvswitch-kmod" - DefaultVlanID = "100" - DefaultOvnLabel = "node-role.kubernetes.io/control-plane" - DefaultDPDKVersion = "19.11" - DefaultDNSAddress = "114.114.114.114" - DefaultDpdkTunnelIface = "br-phy" - DefaultCNIConfigPriority = "01" - DefaultOpenEBSBasePath = "/var/openebs/local" - - Docker = "docker" - Containerd = "containerd" - Crio = "crio" - Isula = "isula" - - Haproxy = "haproxy" - Kubevip = "kube-vip" - DefaultKubeVipMode = "ARP" -) - -func (cfg *ClusterSpec) SetDefaultClusterSpec() (*ClusterSpec, map[string][]*KubeHost) { - clusterCfg := ClusterSpec{} - - clusterCfg.Hosts = SetDefaultHostsCfg(cfg) - clusterCfg.RoleGroups = cfg.RoleGroups - clusterCfg.Etcd = SetDefaultEtcdCfg(cfg) - roleGroups := clusterCfg.GroupHosts() - clusterCfg.ControlPlaneEndpoint = SetDefaultLBCfg(cfg, roleGroups[Master]) - clusterCfg.Network = SetDefaultNetworkCfg(cfg) - clusterCfg.Storage = SetDefaultStorageCfg(cfg) - clusterCfg.System = cfg.System - clusterCfg.Kubernetes = SetDefaultClusterCfg(cfg) - clusterCfg.DNS = cfg.DNS - clusterCfg.Registry = cfg.Registry - clusterCfg.Addons = cfg.Addons - clusterCfg.KubeSphere = cfg.KubeSphere - - if cfg.Kubernetes.ClusterName == "" { - clusterCfg.Kubernetes.ClusterName = DefaultClusterName - } - if cfg.Kubernetes.Version == "" { - clusterCfg.Kubernetes.Version = DefaultKubeVersion - } - if cfg.Kubernetes.MaxPods == 0 { - clusterCfg.Kubernetes.MaxPods = DefaultMaxPods - } - if cfg.Kubernetes.PodPidsLimit == 0 { - clusterCfg.Kubernetes.PodPidsLimit = DefaultPodPidsLimit - } - if cfg.Kubernetes.NodeCidrMaskSize == 0 { - clusterCfg.Kubernetes.NodeCidrMaskSize = DefaultNodeCidrMaskSize - } - if cfg.Kubernetes.ProxyMode == "" { - clusterCfg.Kubernetes.ProxyMode = DefaultProxyMode - } - return &clusterCfg, roleGroups -} - -func SetDefaultHostsCfg(cfg *ClusterSpec) []HostCfg { - var hostCfg []HostCfg - if len(cfg.Hosts) == 0 { - return nil - } - for _, host := range cfg.Hosts { - if len(host.Address) == 0 && len(host.InternalAddress) > 0 { - host.Address = host.InternalAddress - } - if len(host.InternalAddress) == 0 && len(host.Address) > 0 { - host.InternalAddress = host.Address - } - if host.User == "" { - host.User = "root" - } - if host.Port == 0 { - host.Port = DefaultSSHPort - } - if host.PrivateKey == "" { - if host.Password == "" && host.PrivateKeyPath == "" { - host.PrivateKeyPath = "~/.ssh/id_rsa" - } - if host.PrivateKeyPath != "" && strings.HasPrefix(strings.TrimSpace(host.PrivateKeyPath), "~/") { - homeDir, _ := util.Home() - host.PrivateKeyPath = strings.Replace(host.PrivateKeyPath, "~/", fmt.Sprintf("%s/", homeDir), 1) - } - } - - if host.Arch == "" { - host.Arch = DefaultArch - } - - if host.Timeout == nil { - var timeout int64 - timeout = DefaultSSHTimeout - host.Timeout = &timeout - } - - hostCfg = append(hostCfg, host) - } - return hostCfg -} - -func SetDefaultLBCfg(cfg *ClusterSpec, masterGroup []*KubeHost) ControlPlaneEndpoint { - //Check whether LB should be configured - if len(masterGroup) >= 2 && !cfg.ControlPlaneEndpoint.IsInternalLBEnabled() && cfg.ControlPlaneEndpoint.Address == "" && !cfg.ControlPlaneEndpoint.EnableExternalDNS() { - fmt.Println() - fmt.Println("Warning: When there are at least two nodes in the control-plane, you should set the value of the LB address or enable the internal loadbalancer, or set 'controlPlaneEndpoint.externalDNS' to 'true' if the 'controlPlaneEndpoint.domain' can be resolved in your dns server.") - fmt.Println() - } - - // Check whether LB address and the internal LB are both enabled - if cfg.ControlPlaneEndpoint.IsInternalLBEnabled() && cfg.ControlPlaneEndpoint.Address != "" { - fmt.Println("You cannot set up the internal load balancer and the LB address at the same time.") - os.Exit(0) - } - - if (cfg.ControlPlaneEndpoint.Address == "" && !cfg.ControlPlaneEndpoint.EnableExternalDNS()) || cfg.ControlPlaneEndpoint.Address == "127.0.0.1" { - cfg.ControlPlaneEndpoint.Address = masterGroup[0].InternalAddress - } - if cfg.ControlPlaneEndpoint.Domain == "" { - cfg.ControlPlaneEndpoint.Domain = DefaultLBDomain - } - if cfg.ControlPlaneEndpoint.Port == 0 { - cfg.ControlPlaneEndpoint.Port = DefaultLBPort - } - if cfg.ControlPlaneEndpoint.KubeVip.Mode == "" { - cfg.ControlPlaneEndpoint.KubeVip.Mode = DefaultKubeVipMode - } - defaultLbCfg := cfg.ControlPlaneEndpoint - return defaultLbCfg -} - -func SetDefaultNetworkCfg(cfg *ClusterSpec) NetworkConfig { - if cfg.Network.Plugin == "" { - cfg.Network.Plugin = DefaultNetworkPlugin - } - if cfg.Network.KubePodsCIDR == "" { - cfg.Network.KubePodsCIDR = DefaultPodsCIDR - } - if cfg.Network.KubeServiceCIDR == "" { - cfg.Network.KubeServiceCIDR = DefaultServiceCIDR - } - if cfg.Network.Calico.IPIPMode == "" { - cfg.Network.Calico.IPIPMode = DefaultIPIPMode - } - if cfg.Network.Calico.VXLANMode == "" { - cfg.Network.Calico.VXLANMode = DefaultVXLANMode - } - if cfg.Network.Calico.VethMTU == 0 { - cfg.Network.Calico.VethMTU = DefaultVethMTU - } - if cfg.Network.Flannel.BackendMode == "" { - cfg.Network.Flannel.BackendMode = DefaultBackendMode - } - // kube-ovn default config - if cfg.Network.Kubeovn.KubeOvnController.PodGateway == "" { - cfg.Network.Kubeovn.KubeOvnController.PodGateway = DefaultPodGateway - } - if cfg.Network.Kubeovn.JoinCIDR == "" { - cfg.Network.Kubeovn.JoinCIDR = DefaultJoinCIDR - } - if cfg.Network.Kubeovn.Label == "" { - cfg.Network.Kubeovn.Label = DefaultOvnLabel - } - if cfg.Network.Kubeovn.KubeOvnController.VlanID == "" { - cfg.Network.Kubeovn.KubeOvnController.VlanID = DefaultVlanID - } - if cfg.Network.Kubeovn.KubeOvnController.NetworkType == "" { - cfg.Network.Kubeovn.KubeOvnController.NetworkType = DefaultNetworkType - } - if cfg.Network.Kubeovn.TunnelType == "" { - cfg.Network.Kubeovn.TunnelType = DefaultTunnelType - } - if cfg.Network.Kubeovn.KubeOvnController.PodNicType == "" { - cfg.Network.Kubeovn.KubeOvnController.PodNicType = DefaultPodNicType - } - if cfg.Network.Kubeovn.KubeOvnCni.Modules == "" { - cfg.Network.Kubeovn.KubeOvnCni.Modules = DefaultModules - } - if cfg.Network.Kubeovn.KubeOvnCni.RPMs == "" { - cfg.Network.Kubeovn.KubeOvnCni.RPMs = DefaultRPMs - } - if cfg.Network.Kubeovn.KubeOvnPinger.PingerExternalAddress == "" { - cfg.Network.Kubeovn.KubeOvnPinger.PingerExternalAddress = DefaultDNSAddress - } - if cfg.Network.Kubeovn.Dpdk.DpdkVersion == "" { - cfg.Network.Kubeovn.Dpdk.DpdkVersion = DefaultDPDKVersion - } - if cfg.Network.Kubeovn.Dpdk.DpdkTunnelIface == "" { - cfg.Network.Kubeovn.Dpdk.DpdkTunnelIface = DefaultDpdkTunnelIface - } - if cfg.Network.Kubeovn.KubeOvnCni.CNIConfigPriority == "" { - cfg.Network.Kubeovn.KubeOvnCni.CNIConfigPriority = DefaultCNIConfigPriority - } - defaultNetworkCfg := cfg.Network - - return defaultNetworkCfg -} - -func SetDefaultStorageCfg(cfg *ClusterSpec) StorageConfig { - if cfg.Storage.OpenEBS.BasePath == "" { - cfg.Storage.OpenEBS.BasePath = DefaultOpenEBSBasePath - } - defaultStorageCfg := cfg.Storage - return defaultStorageCfg -} - -func SetDefaultClusterCfg(cfg *ClusterSpec) Kubernetes { - if cfg.Kubernetes.Version == "" { - cfg.Kubernetes.Version = DefaultKubeVersion - } else { - s := strings.Split(cfg.Kubernetes.Version, "-") - if len(s) > 1 { - cfg.Kubernetes.Version = s[0] - cfg.Kubernetes.Type = s[1] - } - } - if cfg.Kubernetes.Type == "" { - cfg.Kubernetes.Type = "kubernetes" - } - if cfg.Kubernetes.ClusterName == "" { - cfg.Kubernetes.ClusterName = DefaultClusterName - } - if cfg.Kubernetes.DNSDomain == "" { - cfg.Kubernetes.DNSDomain = DefaultDNSDomain - } - if cfg.Kubernetes.ContainerManager == "" { - cfg.Kubernetes.ContainerManager = Docker - } - if cfg.Kubernetes.ContainerRuntimeEndpoint == "" { - switch cfg.Kubernetes.ContainerManager { - case Docker: - cfg.Kubernetes.ContainerRuntimeEndpoint = "" - case Crio: - cfg.Kubernetes.ContainerRuntimeEndpoint = DefaultCrioEndpoint - case Containerd: - cfg.Kubernetes.ContainerRuntimeEndpoint = DefaultContainerdEndpoint - case Isula: - cfg.Kubernetes.ContainerRuntimeEndpoint = DefaultIsulaEndpoint - default: - cfg.Kubernetes.ContainerRuntimeEndpoint = "" - } - } - defaultClusterCfg := cfg.Kubernetes - - return defaultClusterCfg -} - -func SetDefaultEtcdCfg(cfg *ClusterSpec) EtcdCluster { - if cfg.Etcd.Type == "" || ((cfg.Kubernetes.Type == "k3s" || (len(strings.Split(cfg.Kubernetes.Version, "-")) > 1) && strings.Split(cfg.Kubernetes.Version, "-")[1] == "k3s") && cfg.Etcd.Type == Kubeadm) { - cfg.Etcd.Type = KubeKey - } - if cfg.Etcd.BackupDir == "" { - cfg.Etcd.BackupDir = DefaultEtcdBackupDir - } - if cfg.Etcd.BackupPeriod == 0 { - cfg.Etcd.BackupPeriod = DefaultEtcdBackupPeriod - } - if cfg.Etcd.KeepBackupNumber == 0 { - cfg.Etcd.KeepBackupNumber = DefaultKeepBackNumber - } - if cfg.Etcd.BackupScriptDir == "" { - cfg.Etcd.BackupScriptDir = DefaultEtcdBackupScriptDir - } - - return cfg.Etcd -} diff --git a/cmd/kk/apis/kubekey/v1alpha2/dns_types.go b/cmd/kk/apis/kubekey/v1alpha2/dns_types.go deleted file mode 100644 index aeaf0081d..000000000 --- a/cmd/kk/apis/kubekey/v1alpha2/dns_types.go +++ /dev/null @@ -1,41 +0,0 @@ -/* - Copyright 2023 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1alpha2 - -type DNS struct { - DNSEtcHosts string `yaml:"dnsEtcHosts" json:"dnsEtcHosts"` - CoreDNS CoreDNS `yaml:"coredns" json:"coredns"` - NodeLocalDNS NodeLocalDNS `yaml:"nodelocaldns" json:"nodelocaldns"` -} - -type CoreDNS struct { - AdditionalConfigs string `yaml:"additionalConfigs" json:"additionalConfigs"` - ExternalZones []ExternalZone `yaml:"externalZones" json:"externalZones"` - RewriteBlock string `yaml:"rewriteBlock" json:"rewriteBlock"` - UpstreamDNSServers []string `yaml:"upstreamDNSServers" json:"upstreamDNSServers"` -} - -type NodeLocalDNS struct { - ExternalZones []ExternalZone `yaml:"externalZones" json:"externalZones"` -} - -type ExternalZone struct { - Zones []string `yaml:"zones" json:"zones"` - Nameservers []string `yaml:"nameservers" json:"nameservers"` - Cache int `yaml:"cache" json:"cache"` - Rewrite []string `yaml:"rewrite" json:"rewrite"` -} diff --git a/cmd/kk/apis/kubekey/v1alpha2/etcd_types.go b/cmd/kk/apis/kubekey/v1alpha2/etcd_types.go deleted file mode 100644 index 6dbd30504..000000000 --- a/cmd/kk/apis/kubekey/v1alpha2/etcd_types.go +++ /dev/null @@ -1,59 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1alpha2 - -const ( - KubeKey = "kubekey" - Kubeadm = "kubeadm" - External = "external" -) - -type EtcdCluster struct { - // Type of etcd cluster, can be set to 'kubekey' 'kubeadm' 'external' - Type string `yaml:"type" json:"type,omitempty"` - // ExternalEtcd describes how to connect to an external etcd cluster when type is set to external - External ExternalEtcd `yaml:"external" json:"external,omitempty"` - BackupDir string `yaml:"backupDir" json:"backupDir,omitempty"` - BackupPeriod int `yaml:"backupPeriod" json:"backupPeriod,omitempty"` - KeepBackupNumber int `yaml:"keepBackupNumber" json:"keepBackupNumber,omitempty"` - BackupScriptDir string `yaml:"backupScript" json:"backupScript,omitempty"` - DataDir *string `yaml:"dataDir" json:"dataDir,omitempty"` - HeartbeatInterval *int `yaml:"heartbeatInterval" json:"heartbeatInterval,omitempty"` - ElectionTimeout *int `yaml:"electionTimeout" json:"electionTimeout,omitempty"` - SnapshotCount *int `yaml:"snapshotCount" json:"snapshotCount,omitempty"` - AutoCompactionRetention *int `yaml:"autoCompactionRetention" json:"autoCompactionRetention,omitempty"` - Metrics *string `yaml:"metrics" json:"metrics,omitempty"` - QuotaBackendBytes *int64 `yaml:"quotaBackendBytes" json:"quotaBackendBytes,omitempty"` - MaxRequestBytes *int64 `yaml:"maxRequestBytes" json:"maxRequestBytes,omitempty"` - MaxSnapshots *int `yaml:"maxSnapshots" json:"maxSnapshots,omitempty"` - MaxWals *int `yaml:"maxWals" json:"maxWals,omitempty"` - LogLevel *string `yaml:"logLevel" json:"logLevel"` -} - -// ExternalEtcd describes how to connect to an external etcd cluster -// KubeKey, Kubeadm and External are mutually exclusive -type ExternalEtcd struct { - // Endpoints of etcd members. Useful for using external etcd. - // If not provided, kubeadm will run etcd in a static pod. - Endpoints []string `yaml:"endpoints" json:"endpoints,omitempty"` - // CAFile is an SSL Certificate Authority file used to secure etcd communication. - CAFile string `yaml:"caFile" json:"caFile,omitempty"` - // CertFile is an SSL certification file used to secure etcd communication. - CertFile string `yaml:"certFile" json:"certFile,omitempty"` - // KeyFile is an SSL key file used to secure etcd communication. - KeyFile string `yaml:"keyFile" json:"keyFile,omitempty"` -} diff --git a/cmd/kk/apis/kubekey/v1alpha2/kubernetes_types.go b/cmd/kk/apis/kubekey/v1alpha2/kubernetes_types.go deleted file mode 100644 index c9f055812..000000000 --- a/cmd/kk/apis/kubekey/v1alpha2/kubernetes_types.go +++ /dev/null @@ -1,105 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1alpha2 - -import "k8s.io/apimachinery/pkg/runtime" - -// Kubernetes contains the configuration for the cluster -type Kubernetes struct { - Type string `yaml:"type" json:"type,omitempty"` - Version string `yaml:"version" json:"version,omitempty"` - ClusterName string `yaml:"clusterName" json:"clusterName,omitempty"` - DNSDomain string `yaml:"dnsDomain" json:"dnsDomain,omitempty"` - DisableKubeProxy bool `yaml:"disableKubeProxy" json:"disableKubeProxy,omitempty"` - MasqueradeAll bool `yaml:"masqueradeAll" json:"masqueradeAll,omitempty"` - MaxPods int `yaml:"maxPods" json:"maxPods,omitempty"` - PodPidsLimit int `yaml:"podPidsLimit" json:"podPidsLimit,omitempty"` - NodeCidrMaskSize int `yaml:"nodeCidrMaskSize" json:"nodeCidrMaskSize,omitempty"` - ApiserverCertExtraSans []string `yaml:"apiserverCertExtraSans" json:"apiserverCertExtraSans,omitempty"` - ProxyMode string `yaml:"proxyMode" json:"proxyMode,omitempty"` - AutoRenewCerts *bool `yaml:"autoRenewCerts" json:"autoRenewCerts,omitempty"` - // +optional - Nodelocaldns *bool `yaml:"nodelocaldns" json:"nodelocaldns,omitempty"` - ContainerManager string `yaml:"containerManager" json:"containerManager,omitempty"` - ContainerRuntimeEndpoint string `yaml:"containerRuntimeEndpoint" json:"containerRuntimeEndpoint,omitempty"` - NodeFeatureDiscovery NodeFeatureDiscovery `yaml:"nodeFeatureDiscovery" json:"nodeFeatureDiscovery,omitempty"` - Kata Kata `yaml:"kata" json:"kata,omitempty"` - ApiServerArgs []string `yaml:"apiserverArgs" json:"apiserverArgs,omitempty"` - ControllerManagerArgs []string `yaml:"controllerManagerArgs" json:"controllerManagerArgs,omitempty"` - SchedulerArgs []string `yaml:"schedulerArgs" json:"schedulerArgs,omitempty"` - KubeletArgs []string `yaml:"kubeletArgs" json:"kubeletArgs,omitempty"` - KubeProxyArgs []string `yaml:"kubeProxyArgs" json:"kubeProxyArgs,omitempty"` - FeatureGates map[string]bool `yaml:"featureGates" json:"featureGates,omitempty"` - KubeletConfiguration runtime.RawExtension `yaml:"kubeletConfiguration" json:"kubeletConfiguration,omitempty"` - KubeProxyConfiguration runtime.RawExtension `yaml:"kubeProxyConfiguration" json:"kubeProxyConfiguration,omitempty"` - Audit Audit `yaml:"audit" json:"audit,omitempty"` -} - -// Kata contains the configuration for the kata in cluster -type Kata struct { - Enabled *bool `yaml:"enabled" json:"enabled,omitempty"` -} - -// NodeFeatureDiscovery contains the configuration for the node-feature-discovery in cluster -type NodeFeatureDiscovery struct { - Enabled *bool `yaml:"enabled" json:"enabled,omitempty"` -} - -// Audit contains the configuration for the kube-apiserver audit in cluster -type Audit struct { - Enabled *bool `yaml:"enabled" json:"enabled,omitempty"` -} - -// EnableNodelocaldns is used to determine whether to deploy nodelocaldns. -func (k *Kubernetes) EnableNodelocaldns() bool { - if k.Nodelocaldns == nil { - return true - } - return *k.Nodelocaldns -} - -// EnableKataDeploy is used to determine whether to deploy kata. -func (k *Kubernetes) EnableKataDeploy() bool { - if k.Kata.Enabled == nil { - return false - } - return *k.Kata.Enabled -} - -// EnableNodeFeatureDiscovery is used to determine whether to deploy node-feature-discovery. -func (k *Kubernetes) EnableNodeFeatureDiscovery() bool { - if k.NodeFeatureDiscovery.Enabled == nil { - return false - } - return *k.NodeFeatureDiscovery.Enabled -} - -// EnableAutoRenewCerts is used to determine whether to enable AutoRenewCerts. -func (k *Kubernetes) EnableAutoRenewCerts() bool { - if k.AutoRenewCerts == nil { - return false - } - return *k.AutoRenewCerts -} - -// EnableAudit is used to determine whether to enable kube-apiserver audit. -func (k *Kubernetes) EnableAudit() bool { - if k.Audit.Enabled == nil { - return false - } - return *k.AutoRenewCerts -} diff --git a/cmd/kk/apis/kubekey/v1alpha2/manifest_types.go b/cmd/kk/apis/kubekey/v1alpha2/manifest_types.go deleted file mode 100644 index 66aab5844..000000000 --- a/cmd/kk/apis/kubekey/v1alpha2/manifest_types.go +++ /dev/null @@ -1,120 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha2 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" -) - -type Iso struct { - LocalPath string `yaml:"localPath" json:"localPath"` - Url string `yaml:"url" json:"url"` -} - -type Repository struct { - Iso Iso `yaml:"iso" json:"iso"` -} - -type OperatingSystem struct { - Arch string `yaml:"arch" json:"arch"` - Type string `yaml:"type" json:"type,omitempty"` - Id string `yaml:"id" json:"id"` - Version string `yaml:"version" json:"version"` - OsImage string `yaml:"osImage" json:"osImage"` - Repository Repository `yaml:"repository" json:"repository"` -} - -type KubernetesDistribution struct { - Type string `yaml:"type" json:"type"` - Version string `yaml:"version" json:"version"` -} - -type Helm struct { - Version string `yaml:"version" json:"version"` -} - -type CNI struct { - Version string `yaml:"version" json:"version"` -} - -type ETCD struct { - Version string `yaml:"version" json:"version"` -} - -type DockerManifest struct { - Version string `yaml:"version" json:"version"` -} - -type Crictl struct { - Version string `yaml:"version" json:"version"` -} - -type ContainerRuntime struct { - Type string `yaml:"type" json:"type"` - Version string `yaml:"version" json:"version"` -} - -type DockerRegistry struct { - Version string `yaml:"version" json:"version"` -} - -type Harbor struct { - Version string `yaml:"version" json:"version"` -} - -type DockerCompose struct { - Version string `yaml:"version" json:"version"` -} - -type Calicoctl struct { - Version string `yaml:"version" json:"version"` -} - -type Components struct { - Helm Helm `yaml:"helm" json:"helm"` - CNI CNI `yaml:"cni" json:"cni"` - ETCD ETCD `yaml:"etcd" json:"etcd"` - ContainerRuntimes []ContainerRuntime `yaml:"containerRuntimes" json:"containerRuntimes"` - Crictl Crictl `yaml:"crictl" json:"crictl,omitempty"` - DockerRegistry DockerRegistry `yaml:"docker-registry" json:"docker-registry"` - Harbor Harbor `yaml:"harbor" json:"harbor"` - DockerCompose DockerCompose `yaml:"docker-compose" json:"docker-compose"` - Calicoctl Calicoctl `yaml:"calicoctl" json:"calicoctl"` -} - -type ManifestRegistry struct { - Auths runtime.RawExtension `yaml:"auths" json:"auths,omitempty"` -} - -// ManifestSpec defines the desired state of Manifest -type ManifestSpec struct { - Arches []string `yaml:"arches" json:"arches"` - OperatingSystems []OperatingSystem `yaml:"operatingSystems" json:"operatingSystems"` - KubernetesDistributions []KubernetesDistribution `yaml:"kubernetesDistributions" json:"kubernetesDistributions"` - Components Components `yaml:"components" json:"components"` - Images []string `yaml:"images" json:"images"` - ManifestRegistry ManifestRegistry `yaml:"registry" json:"registry"` -} - -// Manifest is the Schema for the manifests API -type Manifest struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec ManifestSpec `json:"spec,omitempty"` -} diff --git a/cmd/kk/apis/kubekey/v1alpha2/network_types.go b/cmd/kk/apis/kubekey/v1alpha2/network_types.go deleted file mode 100644 index af0b1b6f4..000000000 --- a/cmd/kk/apis/kubekey/v1alpha2/network_types.go +++ /dev/null @@ -1,209 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1alpha2 - -type NetworkConfig struct { - Plugin string `yaml:"plugin" json:"plugin,omitempty"` - KubePodsCIDR string `yaml:"kubePodsCIDR" json:"kubePodsCIDR,omitempty"` - KubeServiceCIDR string `yaml:"kubeServiceCIDR" json:"kubeServiceCIDR,omitempty"` - Calico CalicoCfg `yaml:"calico" json:"calico,omitempty"` - Flannel FlannelCfg `yaml:"flannel" json:"flannel,omitempty"` - Kubeovn KubeovnCfg `yaml:"kubeovn" json:"kubeovn,omitempty"` - MultusCNI MultusCNI `yaml:"multusCNI" json:"multusCNI,omitempty"` - Hybridnet HybridnetCfg `yaml:"hybridnet" json:"hybridnet,omitempty"` -} - -type CalicoCfg struct { - IPIPMode string `yaml:"ipipMode" json:"ipipMode,omitempty"` - VXLANMode string `yaml:"vxlanMode" json:"vxlanMode,omitempty"` - VethMTU int `yaml:"vethMTU" json:"vethMTU,omitempty"` - Ipv4NatOutgoing *bool `yaml:"ipv4NatOutgoing" json:"ipv4NatOutgoing,omitempty"` - DefaultIPPOOL *bool `yaml:"defaultIPPOOL" json:"defaultIPPOOL,omitempty"` - EnableTypha *bool `yaml:"enableTypha" json:"enableTypha,omitempty"` -} - -type FlannelCfg struct { - BackendMode string `yaml:"backendMode" json:"backendMode,omitempty"` - Directrouting bool `yaml:"directRouting" json:"directRouting,omitempty"` -} - -type KubeovnCfg struct { - EnableSSL bool `yaml:"enableSSL" json:"enableSSL,omitempty"` - JoinCIDR string `yaml:"joinCIDR" json:"joinCIDR,omitempty"` - Label string `yaml:"label" json:"label,omitempty"` - TunnelType string `yaml:"tunnelType" json:"tunnelType,omitempty"` - SvcYamlIpfamilypolicy string `yaml:"svcYamlIpfamilypolicy" json:"svcYamlIpfamilypolicy,omitempty"` - Dpdk Dpdk `yaml:"dpdk" json:"dpdk,omitempty"` - OvsOvn OvsOvn `yaml:"ovs-ovn" json:"ovs-ovn,omitempty"` - KubeOvnController KubeOvnController `yaml:"kube-ovn-controller" json:"kube-ovn-controller,omitempty"` - KubeOvnCni KubeOvnCni `yaml:"kube-ovn-cni" json:"kube-ovn-cni,omitempty"` - KubeOvnPinger KubeOvnPinger `yaml:"kube-ovn-pinger" json:"kube-ovn-pinger,omitempty"` -} - -type Dpdk struct { - DpdkMode bool `yaml:"dpdkMode" json:"dpdkMode,omitempty"` - DpdkTunnelIface string `yaml:"dpdkTunnelIface" json:"dpdkTunnelIface,omitempty"` - DpdkVersion string `yaml:"dpdkVersion" json:"dpdkVersion,omitempty"` -} - -type OvsOvn struct { - HwOffload bool `yaml:"hwOffload" json:"hwOffload,omitempty"` -} - -type KubeOvnController struct { - PodGateway string `yaml:"podGateway" json:"podGateway,omitempty"` - CheckGateway *bool `yaml:"checkGateway" json:"checkGateway,omitempty"` - LogicalGateway bool `yaml:"logicalGateway" json:"logicalGateway,omitempty"` - ExcludeIps string `yaml:"excludeIps" json:"excludeIps,omitempty"` - NetworkType string `yaml:"networkType" json:"networkType,omitempty"` - VlanInterfaceName string `yaml:"vlanInterfaceName" json:"vlanInterfaceName,omitempty"` - VlanID string `yaml:"vlanID" json:"vlanID,omitempty"` - PodNicType string `yaml:"podNicType" json:"podNicType,omitempty"` - EnableLB *bool `yaml:"enableLB" json:"enableLB,omitempty"` - EnableNP *bool `yaml:"enableNP" json:"enableNP,omitempty"` - EnableEipSnat *bool `yaml:"enableEipSnat" json:"enableEipSnat,omitempty"` - EnableExternalVPC *bool `yaml:"enableExternalVPC" json:"enableExternalVPC,omitempty"` -} - -type KubeOvnCni struct { - EnableMirror bool `yaml:"enableMirror" json:"enableMirror,omitempty"` - Iface string `yaml:"iface" json:"iface,omitempty"` - CNIConfigPriority string `yaml:"CNIConfigPriority" json:"CNIConfigPriority,omitempty"` - Modules string `yaml:"modules" json:"modules,omitempty"` - RPMs string `yaml:"RPMs" json:"RPMs,omitempty"` -} - -type KubeOvnPinger struct { - PingerExternalAddress string `yaml:"pingerExternalAddress" json:"pingerExternalAddress,omitempty"` - PingerExternalDomain string `yaml:"pingerExternalDomain" json:"pingerExternalDomain,omitempty"` -} - -type HybridnetCfg struct { - DefaultNetworkType string `yaml:"defaultNetworkType" json:"defaultNetworkType,omitempty"` - EnableNetworkPolicy *bool `yaml:"enableNetworkPolicy" json:"enableNetworkPolicy,omitempty"` - Init *bool `yaml:"init" json:"init,omitempty"` - PreferVxlanInterfaces string `yaml:"preferVxlanInterfaces" json:"preferVxlanInterfaces,omitempty"` - PreferVlanInterfaces string `yaml:"preferVlanInterfaces" json:"preferVlanInterfaces,omitempty"` - PreferBGPInterfaces string `yaml:"preferBGPInterfaces" json:"preferBGPInterfaces,omitempty"` - Networks []HybridnetNetwork `yaml:"networks" json:"networks,omitempty"` -} - -type HybridnetNetwork struct { - Name string `yaml:"name" json:"name,omitempty"` - NetID *int `yaml:"netID" json:"netID,omitempty"` - Type string `yaml:"type" json:"type,omitempty"` - Mode string `yaml:"mode" json:"mode,omitempty"` - NodeSelector map[string]string `yaml:"nodeSelector" json:"nodeSelector,omitempty"` - Subnets []HybridnetSubnet `yaml:"subnets" json:"subnets,omitempty"` -} - -type HybridnetSubnet struct { - Name string `yaml:"name" json:"name,omitempty"` - NetID *int `yaml:"netID" json:"netID,omitempty"` - CIDR string `yaml:"cidr" json:"cidr,omitempty"` - Gateway string `yaml:"gateway" json:"gateway,omitempty"` - Start string `yaml:"start" json:"start,omitempty"` - End string `yaml:"end" json:"end,omitempty"` - ReservedIPs []string `yaml:"reservedIPs" json:"reservedIPs,omitempty"` - ExcludeIPs []string `yaml:"excludeIPs" json:"excludeIPs,omitempty"` -} - -func (k *KubeovnCfg) KubeovnCheckGateway() bool { - if k.KubeOvnController.CheckGateway == nil { - return true - } - return *k.KubeOvnController.CheckGateway -} - -func (k *KubeovnCfg) KubeovnEnableLB() bool { - if k.KubeOvnController.EnableLB == nil { - return true - } - return *k.KubeOvnController.EnableLB -} - -func (k *KubeovnCfg) KubeovnEnableNP() bool { - if k.KubeOvnController.EnableNP == nil { - return true - } - return *k.KubeOvnController.EnableNP -} - -func (k *KubeovnCfg) KubeovnEnableEipSnat() bool { - if k.KubeOvnController.EnableEipSnat == nil { - return true - } - return *k.KubeOvnController.EnableEipSnat -} - -func (k *KubeovnCfg) KubeovnEnableExternalVPC() bool { - if k.KubeOvnController.EnableExternalVPC == nil { - return true - } - return *k.KubeOvnController.EnableExternalVPC -} - -type MultusCNI struct { - Enabled *bool `yaml:"enabled" json:"enabled,omitempty"` -} - -func (n *NetworkConfig) EnableMultusCNI() bool { - if n.MultusCNI.Enabled == nil { - return false - } - return *n.MultusCNI.Enabled -} - -// EnableIPV4POOL_NAT_OUTGOING is used to determine whether to enable CALICO_IPV4POOL_NAT_OUTGOING. -func (c *CalicoCfg) EnableIPV4POOL_NAT_OUTGOING() bool { - if c.Ipv4NatOutgoing == nil { - return true - } - return *c.Ipv4NatOutgoing -} - -// EnableDefaultIPPOOL is used to determine whether to create default ippool -func (c *CalicoCfg) EnableDefaultIPPOOL() bool { - if c.DefaultIPPOOL == nil { - return true - } - return *c.DefaultIPPOOL -} - -// Typha is used to determine whether to enable calico Typha -func (c *CalicoCfg) Typha() bool { - if c.EnableTypha == nil { - return false - } - return *c.EnableTypha -} - -// EnableInit is used to determine whether to create default network -func (h *HybridnetCfg) EnableInit() bool { - if h.Init == nil { - return true - } - return *h.Init -} - -// NetworkPolicy is used to determine whether to enable network policy -func (h *HybridnetCfg) NetworkPolicy() bool { - if h.EnableNetworkPolicy == nil { - return true - } - return *h.EnableNetworkPolicy -} diff --git a/cmd/kk/apis/kubekey/v1alpha2/storage_types.go b/cmd/kk/apis/kubekey/v1alpha2/storage_types.go deleted file mode 100644 index a30f51dc4..000000000 --- a/cmd/kk/apis/kubekey/v1alpha2/storage_types.go +++ /dev/null @@ -1,25 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1alpha2 - -type StorageConfig struct { - OpenEBS OpenEBSCfg `yaml:"openebs" json:"openebs,omitempty"` -} - -type OpenEBSCfg struct { - BasePath string `yaml:"basePath" json:"basePath,omitempty"` -} diff --git a/cmd/kk/app/builtin.go b/cmd/kk/app/builtin.go new file mode 100644 index 000000000..e2925379b --- /dev/null +++ b/cmd/kk/app/builtin.go @@ -0,0 +1,32 @@ +//go:build builtin +// +build builtin + +package app + +import ( + "github.com/spf13/cobra" + + "github.com/kubesphere/kubekey/v4/cmd/kk/app/builtin" +) + +// registerInternalCommand registers an internal command to the list of internal commands. +// It ensures that the command is not already registered before adding it to the list. +func registerInternalCommand(command *cobra.Command) { + for _, c := range internalCommand { + if c.Name() == command.Name() { + // command has been registered. skip + return + } + } + internalCommand = append(internalCommand, command) +} + +func init() { + registerInternalCommand(builtin.NewAddCommand()) + registerInternalCommand(builtin.NewArtifactCommand()) + registerInternalCommand(builtin.NewCertsCommand()) + registerInternalCommand(builtin.NewCreateCommand()) + registerInternalCommand(builtin.NewDeleteCommand()) + registerInternalCommand(builtin.NewInitCommand()) + registerInternalCommand(builtin.NewPreCheckCommand()) +} diff --git a/cmd/kk/app/builtin/add.go b/cmd/kk/app/builtin/add.go new file mode 100644 index 000000000..17bb3a0a4 --- /dev/null +++ b/cmd/kk/app/builtin/add.go @@ -0,0 +1,74 @@ +//go:build builtin +// +build builtin + +/* +Copyright 2025 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builtin + +import ( + "github.com/spf13/cobra" + + "github.com/kubesphere/kubekey/v4/cmd/kk/app/options/builtin" +) + +// NewAddCommand creates a new cobra command for adding nodes to a Kubernetes cluster. +// It adds the "nodes" subcommand to enable adding worker nodes. +func NewAddCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "add", + Short: "Add nodes to kubernetes cluster", + } + cmd.AddCommand(newAddNodeCommand()) + + return cmd +} + +// newAddNodeCommand creates a new cobra command for adding worker nodes to an existing cluster. +// It uses the AddNodeOptions to handle configuration and execution of the add nodes operation. +func newAddNodeCommand() *cobra.Command { + o := builtin.NewAddNodeOptions() + + cmd := &cobra.Command{ + Use: "nodes", + Aliases: []string{"node"}, + Short: "Add nodes to the cluster according to the new nodes information from the specified configuration file", + Long: `There are two executors available for adding nodes: + +1. kk add nodes + This will add all nodes listed in the inventory that are not yet installed in the cluster. + +2. kk add nodes --control-plane node1,node2 --worker node1,node2 + This will only add the specified nodes to the cluster as control-plane or worker nodes. + Ensure their connection details are provided in the inventory's hosts file.`, + RunE: func(cmd *cobra.Command, args []string) error { + // Complete the configuration and create a playbook for adding nodes + playbook, err := o.Complete(cmd, []string{"playbooks/add_nodes.yaml"}) + if err != nil { + return err + } + + // Execute the playbook to add the nodes + return o.CommonOptions.Run(cmd.Context(), playbook) + }, + } + flags := cmd.Flags() + for _, f := range o.Flags().FlagSets { + flags.AddFlagSet(f) + } + + return cmd +} diff --git a/cmd/kk/app/builtin/artifact.go b/cmd/kk/app/builtin/artifact.go new file mode 100644 index 000000000..62d5c9e7e --- /dev/null +++ b/cmd/kk/app/builtin/artifact.go @@ -0,0 +1,85 @@ +//go:build builtin +// +build builtin + +/* +Copyright 2024 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builtin + +import ( + "github.com/spf13/cobra" + + "github.com/kubesphere/kubekey/v4/cmd/kk/app/options/builtin" +) + +// NewArtifactCommand creates a new cobra.Command for managing KubeKey offline installation packages. +// It adds subcommands for exporting artifacts and managing images. +func NewArtifactCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "artifact", + Short: "Manage a KubeKey offline installation package", + } + cmd.AddCommand(newArtifactExportCommand()) + cmd.AddCommand(newArtifactImagesCommand()) + + return cmd +} + +func newArtifactExportCommand() *cobra.Command { + o := builtin.NewArtifactExportOptions() + + cmd := &cobra.Command{ + Use: "export", + Short: "Export a KubeKey offline installation package", + RunE: func(cmd *cobra.Command, _ []string) error { + playbook, err := o.Complete(cmd, []string{"playbooks/artifact_export.yaml"}) + if err != nil { + return err + } + + return o.CommonOptions.Run(cmd.Context(), playbook) + }, + } + flags := cmd.Flags() + for _, f := range o.Flags().FlagSets { + flags.AddFlagSet(f) + } + + return cmd +} + +func newArtifactImagesCommand() *cobra.Command { + o := builtin.NewArtifactImagesOptions() + + cmd := &cobra.Command{ + Use: "images", + Short: "push images to a registry from an artifact", + RunE: func(cmd *cobra.Command, _ []string) error { + playbook, err := o.Complete(cmd, []string{"playbooks/artifact_images.yaml"}) + if err != nil { + return err + } + + return o.CommonOptions.Run(cmd.Context(), playbook) + }, + } + flags := cmd.Flags() + for _, f := range o.Flags().FlagSets { + flags.AddFlagSet(f) + } + + return cmd +} diff --git a/cmd/kk/app/builtin/certs.go b/cmd/kk/app/builtin/certs.go new file mode 100644 index 000000000..d5ad3269e --- /dev/null +++ b/cmd/kk/app/builtin/certs.go @@ -0,0 +1,62 @@ +//go:build builtin +// +build builtin + +/* +Copyright 2024 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builtin + +import ( + "github.com/spf13/cobra" + + "github.com/kubesphere/kubekey/v4/cmd/kk/app/options/builtin" +) + +// NewCertsCommand creates a new cobra command for managing cluster certificates. +// It returns a pointer to the created cobra.Command instance. +// The command has a subcommand for renewing certificates. +func NewCertsCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "certs", + Short: "cluster certs", + } + cmd.AddCommand(newCertsRenewCommand()) + + return cmd +} + +func newCertsRenewCommand() *cobra.Command { + o := builtin.NewCertsRenewOptions() + + cmd := &cobra.Command{ + Use: "renew", + Short: "renew a cluster certs", + RunE: func(cmd *cobra.Command, _ []string) error { + playbook, err := o.Complete(cmd, []string{"playbooks/certs_renew.yaml"}) + if err != nil { + return err + } + + return o.CommonOptions.Run(cmd.Context(), playbook) + }, + } + flags := cmd.Flags() + for _, f := range o.Flags().FlagSets { + flags.AddFlagSet(f) + } + + return cmd +} diff --git a/cmd/kk/app/builtin/create.go b/cmd/kk/app/builtin/create.go new file mode 100644 index 000000000..ae26de157 --- /dev/null +++ b/cmd/kk/app/builtin/create.go @@ -0,0 +1,80 @@ +//go:build builtin +// +build builtin + +/* +Copyright 2024 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builtin + +import ( + "github.com/spf13/cobra" + + "github.com/kubesphere/kubekey/v4/cmd/kk/app/options/builtin" +) + +// NewCreateCommand creates a new cobra command for creating a cluster or a cluster configuration file. +// It returns a pointer to the created cobra.Command. +func NewCreateCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "create", + Short: "Create a cluster or a cluster configuration file", + } + cmd.AddCommand(newCreateClusterCommand()) + cmd.AddCommand(newCreateConfigCommand()) + + return cmd +} + +func newCreateClusterCommand() *cobra.Command { + o := builtin.NewCreateClusterOptions() + + cmd := &cobra.Command{ + Use: "cluster", + Short: "Create a Kubernetes or KubeSphere cluster", + RunE: func(cmd *cobra.Command, _ []string) error { + playbook, err := o.Complete(cmd, []string{"playbooks/create_cluster.yaml"}) + if err != nil { + return err + } + + return o.CommonOptions.Run(cmd.Context(), playbook) + }, + } + flags := cmd.Flags() + for _, f := range o.Flags().FlagSets { + flags.AddFlagSet(f) + } + + return cmd +} + +func newCreateConfigCommand() *cobra.Command { + o := builtin.NewCreateConfigOptions() + + cmd := &cobra.Command{ + Use: "config", + Short: "Create a Kubernetes or KubeSphere cluster", + RunE: func(cmd *cobra.Command, _ []string) error { + return o.Run() + }, + } + flags := cmd.Flags() + for _, f := range o.Flags().FlagSets { + flags.AddFlagSet(f) + } + + return cmd +} diff --git a/cmd/kk/app/builtin/delete.go b/cmd/kk/app/builtin/delete.go new file mode 100644 index 000000000..a3ffe5e7e --- /dev/null +++ b/cmd/kk/app/builtin/delete.go @@ -0,0 +1,138 @@ +//go:build builtin +// +build builtin + +/* +Copyright 2025 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builtin + +import ( + "github.com/spf13/cobra" + + "github.com/kubesphere/kubekey/v4/cmd/kk/app/options/builtin" +) + +// NewDeleteCommand creates a new delete command that allows deleting nodes or clusters. +// It provides subcommands for deleting either an entire cluster or individual nodes, as well as the image registry. +func NewDeleteCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "delete", + Short: "Delete node or cluster", + } + // Add subcommands for cluster, nodes, and image registry deletion + cmd.AddCommand(newDeleteClusterCommand()) + cmd.AddCommand(newDeleteNodesCommand()) + cmd.AddCommand(newDeleteRegistryCommand()) + + return cmd +} + +// newDeleteClusterCommand creates a new command for deleting a Kubernetes cluster. +// It uses the delete_cluster.yaml playbook to: +// - Uninstall Kubernetes components from all nodes +// - Clean up container runtime if specified +// - Remove DNS entries if specified +// - Clean up etcd if specified +func newDeleteClusterCommand() *cobra.Command { + // Initialize options for deleting a cluster + o := builtin.NewDeleteClusterOptions() + + cmd := &cobra.Command{ + Use: "cluster", + Short: "Delete a cluster", + RunE: func(cmd *cobra.Command, args []string) error { + // Complete the configuration and create a playbook for deleting the cluster + playbook, err := o.Complete(cmd, []string{"playbooks/delete_cluster.yaml"}) + if err != nil { + return err + } + + // Execute the playbook to delete the cluster + return o.CommonOptions.Run(cmd.Context(), playbook) + }, + } + // Add all relevant flag sets to the command + flags := cmd.Flags() + for _, f := range o.Flags().FlagSets { + flags.AddFlagSet(f) + } + + return cmd +} + +// newDeleteNodesCommand creates a new command for deleting specific nodes from a cluster. +// It uses the delete_nodes.yaml playbook to: +// - Remove Kubernetes components from specified nodes +// - Clean up container runtime if specified +// - Remove DNS entries if specified +// - Clean up etcd if the node was part of etcd cluster +func newDeleteNodesCommand() *cobra.Command { + // Initialize options for deleting nodes + o := builtin.NewDeleteNodesOptions() + + cmd := &cobra.Command{ + Use: "nodes {node1 node2 ...}", + Aliases: []string{"node"}, + Short: "Delete a cluster nodes", + RunE: func(cmd *cobra.Command, args []string) error { + // Complete the configuration and create a playbook for deleting nodes + // The playbook path is appended as the last argument + playbook, err := o.Complete(cmd, append(args, "playbooks/delete_nodes.yaml")) + if err != nil { + return err + } + + // Execute the playbook to delete the specified nodes + return o.CommonOptions.Run(cmd.Context(), playbook) + }, + } + // Add all relevant flag sets to the command + flags := cmd.Flags() + for _, f := range o.Flags().FlagSets { + flags.AddFlagSet(f) + } + + return cmd +} + +// newDeleteRegistryCommand creates a new command for deleting the image registry created by kubekey. +// It uses the delete_image_registry.yaml playbook to remove the image registry and optionally its container runtime. +func newDeleteRegistryCommand() *cobra.Command { + // Initialize options for deleting the image registry + o := builtin.NewDeleteRegistryOptions() + + cmd := &cobra.Command{ + Use: "registry", + Short: "Delete a image registry which create by kubekey.", + RunE: func(cmd *cobra.Command, args []string) error { + // Complete the configuration and create a playbook for deleting the image registry + playbook, err := o.Complete(cmd, []string{"playbooks/delete_registry.yaml"}) + if err != nil { + return err + } + + // Execute the playbook to delete the image registry + return o.CommonOptions.Run(cmd.Context(), playbook) + }, + } + // Add all relevant flag sets to the command + flags := cmd.Flags() + for _, f := range o.Flags().FlagSets { + flags.AddFlagSet(f) + } + + return cmd +} diff --git a/cmd/kk/app/builtin/init.go b/cmd/kk/app/builtin/init.go new file mode 100644 index 000000000..4ccdea860 --- /dev/null +++ b/cmd/kk/app/builtin/init.go @@ -0,0 +1,86 @@ +//go:build builtin +// +build builtin + +/* +Copyright 2024 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builtin + +import ( + "github.com/spf13/cobra" + + "github.com/kubesphere/kubekey/v4/cmd/kk/app/options/builtin" +) + +// NewInitCommand creates a new cobra.Command for initializing the installation environment. +// It sets up the "init" command with a short description and adds subcommands for initializing +// the operating system and the registry. +func NewInitCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "init", + Short: "Initializes the installation environment", + } + cmd.AddCommand(newInitOSCommand()) + cmd.AddCommand(newInitRegistryCommand()) + + return cmd +} + +func newInitOSCommand() *cobra.Command { + o := builtin.NewInitOSOptions() + + cmd := &cobra.Command{ + Use: "os", + Short: "Init operating system", + RunE: func(cmd *cobra.Command, _ []string) error { + playbook, err := o.Complete(cmd, []string{"playbooks/init_os.yaml"}) + if err != nil { + return err + } + + return o.CommonOptions.Run(cmd.Context(), playbook) + }, + } + flags := cmd.Flags() + for _, f := range o.Flags().FlagSets { + flags.AddFlagSet(f) + } + + return cmd +} + +func newInitRegistryCommand() *cobra.Command { + o := builtin.NewInitRegistryOptions() + + cmd := &cobra.Command{ + Use: "registry", + Short: "Init a local image registry", + RunE: func(cmd *cobra.Command, _ []string) error { + playbook, err := o.Complete(cmd, []string{"playbooks/init_registry.yaml"}) + if err != nil { + return err + } + + return o.CommonOptions.Run(cmd.Context(), playbook) + }, + } + flags := cmd.Flags() + for _, f := range o.Flags().FlagSets { + flags.AddFlagSet(f) + } + + return cmd +} diff --git a/cmd/kk/app/builtin/precheck.go b/cmd/kk/app/builtin/precheck.go new file mode 100644 index 000000000..4d7b485ce --- /dev/null +++ b/cmd/kk/app/builtin/precheck.go @@ -0,0 +1,57 @@ +//go:build builtin +// +build builtin + +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builtin + +import ( + "github.com/spf13/cobra" + + "github.com/kubesphere/kubekey/v4/cmd/kk/app/options/builtin" +) + +// NewPreCheckCommand creates a new cobra.Command for performing pre-checks on nodes +// to determine their eligibility for cluster deployment. The command supports +// specifying tags to check specific items such as etcd, os, network, cri, and nfs. +// +// Returns: +// +// *cobra.Command: The cobra command configured for pre-checks. +func NewPreCheckCommand() *cobra.Command { + o := builtin.NewPreCheckOptions() + + cmd := &cobra.Command{ + Use: "precheck tags...", + Short: "Check if the nodes is eligible for cluster deployment.", + Long: "the tags can specify check items. support: etcd, os, network, cri, nfs.", + RunE: func(cmd *cobra.Command, args []string) error { + playbook, err := o.Complete(cmd, append(args, "playbooks/precheck.yaml")) + if err != nil { + return err + } + + return o.CommonOptions.Run(cmd.Context(), playbook) + }, + } + flags := cmd.Flags() + for _, f := range o.Flags().FlagSets { + flags.AddFlagSet(f) + } + + return cmd +} diff --git a/cmd/kk/app/options/builtin/add.go b/cmd/kk/app/options/builtin/add.go new file mode 100644 index 000000000..97192e205 --- /dev/null +++ b/cmd/kk/app/options/builtin/add.go @@ -0,0 +1,146 @@ +//go:build builtin +// +build builtin + +/* +Copyright 2025 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builtin + +import ( + "fmt" + "slices" + "strings" + + "github.com/cockroachdb/errors" + kkcorev1 "github.com/kubesphere/kubekey/api/core/v1" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + cliflag "k8s.io/component-base/cli/flag" + + "github.com/kubesphere/kubekey/v4/cmd/kk/app/options" + _const "github.com/kubesphere/kubekey/v4/pkg/const" + "github.com/kubesphere/kubekey/v4/pkg/variable" +) + +// NewAddNodeOptions creates a new AddNodeOptions with default values +func NewAddNodeOptions() *AddNodeOptions { + // set default value + o := &AddNodeOptions{ + CommonOptions: options.NewCommonOptions(), + Kubernetes: defaultKubeVersion, + } + o.CommonOptions.GetInventoryFunc = getInventory + + return o +} + +// AddNodeOptions contains options for adding nodes to a cluster +type AddNodeOptions struct { + options.CommonOptions + // kubernetes version which the cluster will install. + Kubernetes string + // ControlPlane nodes which will be added. + ControlPlane string + // Worker nodes which will to be added. + Worker string +} + +// Flags adds flags for configuring AddNodeOptions to the specified FlagSet +func (o *AddNodeOptions) Flags() cliflag.NamedFlagSets { + fss := o.CommonOptions.Flags() + kfs := fss.FlagSet("config") + kfs.StringVar(&o.Kubernetes, "with-kubernetes", o.Kubernetes, fmt.Sprintf("Specify a supported version of kubernetes. default is %s", o.Kubernetes)) + kfs.StringVar(&o.ControlPlane, "control-plane", o.ControlPlane, "Which nodes will be installed as control-plane. Multiple nodes are supported, separated by commas (e.g., node1, node2, ...)") + kfs.StringVar(&o.Worker, "worker", o.Worker, "Which nodes will be installed as workers. Multiple nodes are supported, separated by commas (e.g., node1, node2, ...)") + + return fss +} + +// Complete validates and completes the AddNodeOptions configuration. +// It creates and returns a Playbook object based on the options. +func (o *AddNodeOptions) Complete(cmd *cobra.Command, args []string) (*kkcorev1.Playbook, error) { + playbook := &kkcorev1.Playbook{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "add-nodes-", + Namespace: o.Namespace, + Annotations: map[string]string{ + kkcorev1.BuiltinsProjectAnnotation: "", + }, + }, + } + + // complete playbook. now only support one playbook + if len(args) != 1 { + return nil, errors.Errorf("%s\nSee '%s -h' for help and examples", cmd.Use, cmd.CommandPath()) + } + o.Playbook = args[0] + + playbook.Spec = kkcorev1.PlaybookSpec{ + Playbook: o.Playbook, + } + // override kube_version in config + if err := o.CommonOptions.Complete(playbook); err != nil { + return nil, err + } + + return playbook, o.complete() +} + +// complete updates the configuration with container manager and kubernetes version settings +func (o *AddNodeOptions) complete() error { + if _, ok, _ := unstructured.NestedFieldNoCopy(o.CommonOptions.Config.Value(), "kubernetes", "kube_version"); !ok { + if err := unstructured.SetNestedField(o.CommonOptions.Config.Value(), o.Kubernetes, "kubernetes", "kube_version"); err != nil { + return errors.Wrapf(err, "failed to set %q to config", "kube_version") + } + } + + var addNodes []string + groups := variable.ConvertGroup(*o.Inventory) + // add nodes to control_plane group + if o.ControlPlane != "" { + for _, node := range strings.Split(o.ControlPlane, ",") { + if !slices.Contains(groups[_const.VariableGroupsAll], node) { + return errors.Errorf("%q is not defined in inventory.", node) + } + if !slices.Contains(groups[defaultGroupControlPlane], node) { + group := o.Inventory.Spec.Groups[defaultGroupControlPlane] + group.Hosts = append(group.Hosts, node) + o.Inventory.Spec.Groups[defaultGroupControlPlane] = group + } + addNodes = append(addNodes, node) + } + } + // add nodes to worker group + if o.Worker != "" { + for _, node := range strings.Split(o.ControlPlane, ",") { + if !slices.Contains(groups[_const.VariableGroupsAll], node) { + return errors.Errorf("%q is not defined in inventory.", node) + } + if !slices.Contains(groups[defaultGroupWorker], node) { + group := o.Inventory.Spec.Groups[defaultGroupWorker] + group.Hosts = append(group.Hosts, node) + o.Inventory.Spec.Groups[defaultGroupControlPlane] = group + } + addNodes = append(addNodes, node) + } + } + if err := unstructured.SetNestedStringSlice(o.CommonOptions.Config.Value(), addNodes, "add_nodes"); err != nil { + return errors.Wrapf(err, "failed to set %q to config", "add_nodes") + } + + return nil +} diff --git a/cmd/kk/app/options/builtin/artifact.go b/cmd/kk/app/options/builtin/artifact.go new file mode 100644 index 000000000..38aa4e025 --- /dev/null +++ b/cmd/kk/app/options/builtin/artifact.go @@ -0,0 +1,160 @@ +//go:build builtin +// +build builtin + +/* +Copyright 2024 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builtin + +import ( + "fmt" + + "github.com/cockroachdb/errors" + kkcorev1 "github.com/kubesphere/kubekey/api/core/v1" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + cliflag "k8s.io/component-base/cli/flag" + + "github.com/kubesphere/kubekey/v4/cmd/kk/app/options" +) + +// ====================================================================================== +// artifact export +// ====================================================================================== + +// ArtifactExportOptions for NewArtifactExportOptions +type ArtifactExportOptions struct { + options.CommonOptions + // kubernetes version which the cluster will install. + Kubernetes string +} + +// NewArtifactExportOptions for newArtifactExportCommand +func NewArtifactExportOptions() *ArtifactExportOptions { + // set default value + o := &ArtifactExportOptions{CommonOptions: options.NewCommonOptions()} + o.CommonOptions.GetInventoryFunc = getInventory + + return o +} + +// Flags add to newArtifactExportCommand +func (o *ArtifactExportOptions) Flags() cliflag.NamedFlagSets { + fss := o.CommonOptions.Flags() + kfs := fss.FlagSet("config") + kfs.StringVar(&o.Kubernetes, "with-kubernetes", o.Kubernetes, fmt.Sprintf("Specify a supported version of kubernetes. default is %s", o.Kubernetes)) + + return fss +} + +// Complete options. create Playbook, Config and Inventory +func (o *ArtifactExportOptions) Complete(cmd *cobra.Command, args []string) (*kkcorev1.Playbook, error) { + playbook := &kkcorev1.Playbook{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "artifact-export-", + Namespace: o.Namespace, + Annotations: map[string]string{ + kkcorev1.BuiltinsProjectAnnotation: "", + }, + }, + } + + // complete playbook. now only support one playbook + if len(args) != 1 { + return nil, errors.Errorf("%s\nSee '%s -h' for help and examples", cmd.Use, cmd.CommandPath()) + } + o.Playbook = args[0] + + playbook.Spec = kkcorev1.PlaybookSpec{ + Playbook: o.Playbook, + SkipTags: []string{"certs"}, + } + if err := o.CommonOptions.Complete(playbook); err != nil { + return nil, err + } + + return playbook, nil +} + +// ====================================================================================== +// artifact image +// ====================================================================================== + +// ArtifactImagesOptions for NewArtifactImagesOptions +type ArtifactImagesOptions struct { + options.CommonOptions + // kubernetes version which the cluster will install. + Kubernetes string + Push bool + Pull bool +} + +// NewArtifactImagesOptions for newArtifactImagesCommand +func NewArtifactImagesOptions() *ArtifactImagesOptions { + o := &ArtifactImagesOptions{CommonOptions: options.NewCommonOptions()} + o.CommonOptions.GetInventoryFunc = getInventory + + return o +} + +// Flags add to newArtifactImagesCommand +func (o *ArtifactImagesOptions) Flags() cliflag.NamedFlagSets { + fss := o.CommonOptions.Flags() + kfs := fss.FlagSet("config") + kfs.StringVar(&o.Kubernetes, "with-kubernetes", o.Kubernetes, fmt.Sprintf("Specify a supported version of kubernetes. default is %s", o.Kubernetes)) + kfs.BoolVar(&o.Push, "push", o.Push, "Push image to image registry") + kfs.BoolVar(&o.Pull, "pull", o.Pull, "Pull image to binary dir") + + return fss +} + +// Complete options. create Playbook, Config and Inventory +func (o *ArtifactImagesOptions) Complete(cmd *cobra.Command, args []string) (*kkcorev1.Playbook, error) { + playbook := &kkcorev1.Playbook{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "artifact-images-", + Namespace: o.Namespace, + Annotations: map[string]string{ + kkcorev1.BuiltinsProjectAnnotation: "", + }, + }, + } + + // complete playbook. now only support one playbook + if len(args) != 1 { + return nil, errors.Errorf("%s\nSee '%s -h' for help and examples", cmd.Use, cmd.CommandPath()) + } + o.Playbook = args[0] + + var tags = []string{"image_registry"} + if o.Push { + tags = append(tags, "push") + } + if o.Pull { + tags = append(tags, "pull") + } + + playbook.Spec = kkcorev1.PlaybookSpec{ + Playbook: o.Playbook, + Tags: tags, + } + + if err := o.CommonOptions.Complete(playbook); err != nil { + return nil, errors.WithStack(err) + } + + return playbook, nil +} diff --git a/cmd/kk/app/options/builtin/builtin.go b/cmd/kk/app/options/builtin/builtin.go new file mode 100644 index 000000000..1fa5f8031 --- /dev/null +++ b/cmd/kk/app/options/builtin/builtin.go @@ -0,0 +1,65 @@ +//go:build builtin +// +build builtin + +/* +Copyright 2024 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builtin + +import ( + "bytes" + "fmt" + "text/template" + + "github.com/cockroachdb/errors" + kkcorev1 "github.com/kubesphere/kubekey/api/core/v1" + "k8s.io/apimachinery/pkg/util/yaml" + + "github.com/kubesphere/kubekey/v4/builtin/core" + "github.com/kubesphere/kubekey/v4/cmd/kk/app/options" +) + +const ( + defaultKubeVersion = "v1.33.1" +) + +const ( + defaultGroupControlPlane = "kube_control_plane" + defaultGroupWorker = "kube_worker" +) + +var getInventory options.InventoryFunc = func() (*kkcorev1.Inventory, error) { + data, err := core.Defaults.ReadFile("defaults/inventory/localhost.yaml") + if err != nil { + return nil, errors.Wrap(err, "failed to get local inventory. Please set it by \"--inventory\"") + } + + inventory := &kkcorev1.Inventory{} + return inventory, errors.Wrap(yaml.Unmarshal(data, inventory), "failed to unmarshal local inventory file: %q.") +} + +func getConfig(kubeVersion string) ([]byte, error) { + t, err := template.ParseFS(core.Defaults, fmt.Sprintf("defaults/config/%s.yaml", kubeVersion[:5])) + if err != nil { + return nil, errors.Wrapf(err, "failed to get local configFile template for kube_version: %q. Please set it by \"--config\"", kubeVersion) + } + data := bytes.NewBuffer(nil) + if err := t.Execute(data, map[string]string{"kube_version": kubeVersion}); err != nil { + return nil, errors.Wrapf(err, "failed to parse local configFile template for kube_version: %q. Please set it by \"--config\"", kubeVersion) + } + + return data.Bytes(), nil +} diff --git a/cmd/kk/app/options/builtin/certs.go b/cmd/kk/app/options/builtin/certs.go new file mode 100644 index 000000000..3580bb3a8 --- /dev/null +++ b/cmd/kk/app/options/builtin/certs.go @@ -0,0 +1,81 @@ +//go:build builtin +// +build builtin + +/* +Copyright 2024 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builtin + +import ( + "fmt" + + "github.com/cockroachdb/errors" + kkcorev1 "github.com/kubesphere/kubekey/api/core/v1" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + cliflag "k8s.io/component-base/cli/flag" + + "github.com/kubesphere/kubekey/v4/cmd/kk/app/options" +) + +// NewCertsRenewOptions for newCertsRenewCommand +func NewCertsRenewOptions() *CertsRenewOptions { + // set default value + o := &CertsRenewOptions{CommonOptions: options.NewCommonOptions()} + o.CommonOptions.GetInventoryFunc = getInventory + + return o +} + +// CertsRenewOptions for NewCertsRenewOptions +type CertsRenewOptions struct { + options.CommonOptions + // kubernetes version which the cluster will install. + Kubernetes string +} + +// Flags add to newCertsRenewCommand +func (o *CertsRenewOptions) Flags() cliflag.NamedFlagSets { + fss := o.CommonOptions.Flags() + kfs := fss.FlagSet("config") + kfs.StringVar(&o.Kubernetes, "with-kubernetes", o.Kubernetes, fmt.Sprintf("Specify a supported version of kubernetes. default is %s", o.Kubernetes)) + + return fss +} + +// Complete options. create Playbook, Config and Inventory +func (o *CertsRenewOptions) Complete(cmd *cobra.Command, args []string) (*kkcorev1.Playbook, error) { + playbook := &kkcorev1.Playbook{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "certs-renew-", + Namespace: o.Namespace, + Annotations: map[string]string{ + kkcorev1.BuiltinsProjectAnnotation: "", + }, + }, + } + // complete playbook. now only support one playbook + if len(args) != 1 { + return nil, errors.Errorf("%s\nSee '%s -h' for help and examples", cmd.Use, cmd.CommandPath()) + } + o.Playbook = args[0] + playbook.Spec = kkcorev1.PlaybookSpec{ + Playbook: o.Playbook, + Tags: []string{"certs"}, + } + + return playbook, o.CommonOptions.Complete(playbook) +} diff --git a/cmd/kk/app/options/builtin/create.go b/cmd/kk/app/options/builtin/create.go new file mode 100644 index 000000000..63d06af97 --- /dev/null +++ b/cmd/kk/app/options/builtin/create.go @@ -0,0 +1,159 @@ +//go:build builtin +// +build builtin + +/* +Copyright 2024 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builtin + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/cockroachdb/errors" + kkcorev1 "github.com/kubesphere/kubekey/api/core/v1" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + cliflag "k8s.io/component-base/cli/flag" + + "github.com/kubesphere/kubekey/v4/cmd/kk/app/options" +) + +// ====================================================================================== +// create cluster +// ====================================================================================== + +// NewCreateClusterOptions for newCreateClusterCommand +func NewCreateClusterOptions() *CreateClusterOptions { + // set default value + o := &CreateClusterOptions{ + CommonOptions: options.NewCommonOptions(), + Kubernetes: defaultKubeVersion, + } + o.CommonOptions.GetInventoryFunc = getInventory + + return o +} + +// CreateClusterOptions for NewCreateClusterOptions +type CreateClusterOptions struct { + options.CommonOptions + // kubernetes version which the cluster will install. + Kubernetes string +} + +// Flags add to newCreateClusterCommand +func (o *CreateClusterOptions) Flags() cliflag.NamedFlagSets { + fss := o.CommonOptions.Flags() + kfs := fss.FlagSet("config") + kfs.StringVar(&o.Kubernetes, "with-kubernetes", o.Kubernetes, fmt.Sprintf("Specify a supported version of kubernetes. default is %s", o.Kubernetes)) + + return fss +} + +// Complete options. create Playbook, Config and Inventory +func (o *CreateClusterOptions) Complete(cmd *cobra.Command, args []string) (*kkcorev1.Playbook, error) { + playbook := &kkcorev1.Playbook{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "create-cluster-", + Namespace: o.Namespace, + Annotations: map[string]string{ + kkcorev1.BuiltinsProjectAnnotation: "", + }, + }, + } + + // complete playbook. now only support one playbook + if len(args) != 1 { + return nil, errors.Errorf("%s\nSee '%s -h' for help and examples", cmd.Use, cmd.CommandPath()) + } + o.Playbook = args[0] + + playbook.Spec = kkcorev1.PlaybookSpec{ + Playbook: o.Playbook, + } + if err := o.CommonOptions.Complete(playbook); err != nil { + return nil, err + } + + return playbook, o.completeConfig() +} + +func (o *CreateClusterOptions) completeConfig() error { + if _, ok, _ := unstructured.NestedFieldNoCopy(o.CommonOptions.Config.Value(), "kubernetes", "kube_version"); !ok { + if err := unstructured.SetNestedField(o.CommonOptions.Config.Value(), o.Kubernetes, "kubernetes", "kube_version"); err != nil { + return errors.Wrapf(err, "failed to set %q to config", "kube_version") + } + } + + return nil +} + +// ====================================================================================== +// create config +// ====================================================================================== + +// NewCreateConfigOptions for newCreateConfigCommand +func NewCreateConfigOptions() *CreateConfigOptions { + // set default value + return &CreateConfigOptions{ + Kubernetes: defaultKubeVersion, + } +} + +// CreateConfigOptions for NewCreateConfigOptions +type CreateConfigOptions struct { + // kubernetes version which the config will install. + Kubernetes string + // OutputDir for config file. if set will generate file in this dir + OutputDir string +} + +// Flags add to newCreateConfigCommand +func (o *CreateConfigOptions) Flags() cliflag.NamedFlagSets { + fss := cliflag.NamedFlagSets{} + kfs := fss.FlagSet("config") + kfs.StringVar(&o.Kubernetes, "with-kubernetes", o.Kubernetes, fmt.Sprintf("Specify a supported version of kubernetes. default is %s", o.Kubernetes)) + kfs.StringVarP(&o.OutputDir, "output", "o", o.OutputDir, "Output dir for config. if not set will output to stdout") + + return fss +} + +// Run executes the create config operation. It reads the default config file for the specified +// Kubernetes version and either writes it to the specified output directory or prints it to stdout. +// If an output directory is specified, it creates a config file named "config-.yaml". +func (o *CreateConfigOptions) Run() error { + // Read the default config file for the specified Kubernetes version + data, err := getConfig(o.Kubernetes) + if err != nil { + return err + } + if o.OutputDir != "" { + // Write config to file if output directory is specified + filename := filepath.Join(o.OutputDir, fmt.Sprintf("config-%s.yaml", o.Kubernetes)) + if err := os.WriteFile(filename, data, os.ModePerm); err != nil { + return errors.Wrapf(err, "failed to write config file to %s", filename) + } + fmt.Printf("write config file to %s success.\n", filename) + } else { + // Print config to stdout if no output directory specified + fmt.Println(string(data)) + } + + return nil +} diff --git a/cmd/kk/app/options/builtin/delete.go b/cmd/kk/app/options/builtin/delete.go new file mode 100644 index 000000000..0bac9eaae --- /dev/null +++ b/cmd/kk/app/options/builtin/delete.go @@ -0,0 +1,263 @@ +//go:build builtin +// +build builtin + +/* +Copyright 2025 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builtin + +import ( + "fmt" + + "github.com/cockroachdb/errors" + kkcorev1 "github.com/kubesphere/kubekey/api/core/v1" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + cliflag "k8s.io/component-base/cli/flag" + + "github.com/kubesphere/kubekey/v4/cmd/kk/app/options" +) + +// ====================================================================================== +// delete cluster +// ====================================================================================== + +// NewDeleteClusterOptions creates a new DeleteClusterOptions with default values +func NewDeleteClusterOptions() *DeleteClusterOptions { + // set default value for DeleteClusterOptions + o := &DeleteClusterOptions{ + CommonOptions: options.NewCommonOptions(), + Kubernetes: defaultKubeVersion, + } + // Set the function to get the config for the specified Kubernetes version + // Set the function to get the inventory + o.CommonOptions.GetInventoryFunc = getInventory + + return o +} + +// DeleteClusterOptions contains options for deleting a Kubernetes cluster +type DeleteClusterOptions struct { + options.CommonOptions + // Kubernetes version which the cluster will install. + Kubernetes string +} + +// Flags returns the flag sets for DeleteClusterOptions +func (o *DeleteClusterOptions) Flags() cliflag.NamedFlagSets { + fss := o.CommonOptions.Flags() + kfs := fss.FlagSet("config") + // Add a flag for specifying the Kubernetes version + kfs.StringVar(&o.Kubernetes, "with-kubernetes", o.Kubernetes, fmt.Sprintf("Specify a supported version of kubernetes. default is %s", o.Kubernetes)) + + return fss +} + +// Complete validates and completes the DeleteClusterOptions configuration +func (o *DeleteClusterOptions) Complete(cmd *cobra.Command, args []string) (*kkcorev1.Playbook, error) { + // Initialize playbook metadata for deleting a cluster + playbook := &kkcorev1.Playbook{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "delete-cluster-", + Namespace: o.Namespace, + Annotations: map[string]string{ + kkcorev1.BuiltinsProjectAnnotation: "", + }, + }, + } + + // Validate playbook arguments: must have exactly one argument (the playbook) + if len(args) != 1 { + return nil, errors.Errorf("%s\nSee '%s -h' for help and examples", cmd.Use, cmd.CommandPath()) + } + o.Playbook = args[0] + + // Set playbook specification + playbook.Spec = kkcorev1.PlaybookSpec{ + Playbook: o.Playbook, + } + + // Complete common options (e.g., config, inventory) + if err := o.CommonOptions.Complete(playbook); err != nil { + return nil, err + } + + // Complete config specific to delete cluster + return playbook, o.completeConfig() +} + +// completeConfig updates the configuration with container manager settings +func (o *DeleteClusterOptions) completeConfig() error { + // If kube_version is not set in config, set it to the specified Kubernetes version + if _, ok, _ := unstructured.NestedFieldNoCopy(o.CommonOptions.Config.Value(), "kubernetes", "kube_version"); !ok { + if err := unstructured.SetNestedField(o.CommonOptions.Config.Value(), o.Kubernetes, "kubernetes", "kube_version"); err != nil { + return errors.Wrapf(err, "failed to set %q to config", "kube_version") + } + } + + return nil +} + +// ====================================================================================== +// delete nodes +// ====================================================================================== + +// NewDeleteNodesOptions creates a new DeleteNodesOptions with default values +func NewDeleteNodesOptions() *DeleteNodesOptions { + // set default value for DeleteNodesOptions + o := &DeleteNodesOptions{ + CommonOptions: options.NewCommonOptions(), + Kubernetes: defaultKubeVersion, + } + // Set the function to get the inventory + o.CommonOptions.GetInventoryFunc = getInventory + + return o +} + +// DeleteNodesOptions contains options for deleting Kubernetes cluster nodes +type DeleteNodesOptions struct { + options.CommonOptions + // Kubernetes version which the cluster will install. + Kubernetes string +} + +// Flags returns the flag sets for DeleteNodesOptions +func (o *DeleteNodesOptions) Flags() cliflag.NamedFlagSets { + fss := o.CommonOptions.Flags() + kfs := fss.FlagSet("config") + // Add a flag for specifying the Kubernetes version + kfs.StringVar(&o.Kubernetes, "with-kubernetes", o.Kubernetes, fmt.Sprintf("Specify a supported version of kubernetes. default is %s", o.Kubernetes)) + + return fss +} + +// Complete validates and completes the DeleteNodesOptions configuration +func (o *DeleteNodesOptions) Complete(cmd *cobra.Command, args []string) (*kkcorev1.Playbook, error) { + // Initialize playbook metadata for deleting nodes + playbook := &kkcorev1.Playbook{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "delete-nodes-", + Namespace: o.Namespace, + Annotations: map[string]string{ + kkcorev1.BuiltinsProjectAnnotation: "", + }, + }, + } + + // Validate playbook arguments: must have at least one argument (nodes + playbook) + if len(args) < 1 { + return nil, errors.Errorf("%s\nSee '%s -h' for help and examples", cmd.Use, cmd.CommandPath()) + } + // All arguments except the last are node names, the last is the playbook + nodes := args[:len(args)-1] + o.Playbook = args[len(args)-1] + + // Set playbook specification + playbook.Spec = kkcorev1.PlaybookSpec{ + Playbook: o.Playbook, + } + + // Complete common options (e.g., config, inventory) + if err := o.CommonOptions.Complete(playbook); err != nil { + return nil, err + } + + // Complete config specific to delete nodes + return playbook, o.completeConfig(nodes) +} + +// completeConfig updates the configuration with container manager settings +func (o *DeleteNodesOptions) completeConfig(nodes []string) error { + // If kube_version is not set in config, set it to the specified Kubernetes version + if _, ok, _ := unstructured.NestedFieldNoCopy(o.CommonOptions.Config.Value(), "kubernetes", "kube_version"); !ok { + if err := unstructured.SetNestedField(o.CommonOptions.Config.Value(), o.Kubernetes, "kubernetes", "kube_version"); err != nil { + return errors.Wrapf(err, "failed to set %q to config", "kube_version") + } + } + // Set the list of nodes to be deleted in the config + if err := unstructured.SetNestedStringSlice(o.CommonOptions.Config.Value(), nodes, "delete_nodes"); err != nil { + return errors.Wrapf(err, "failed to set %q to config", "delete_nodes") + } + + return nil +} + +// ====================================================================================== +// delete registry +// ====================================================================================== + +// NewDeleteRegistryOptions creates a new DeleteRegistryOptions with default values +func NewDeleteRegistryOptions() *DeleteRegistryOptions { + // set default value for DeleteImageRegistryOptions + o := &DeleteRegistryOptions{ + CommonOptions: options.NewCommonOptions(), + } + // Set the function to get the inventory + o.CommonOptions.GetInventoryFunc = getInventory + + return o +} + +// DeleteRegistryOptions contains options for deleting an image_registry created by kubekey +type DeleteRegistryOptions struct { + options.CommonOptions + // kubernetes version which the config will install. + Kubernetes string +} + +// Flags returns the flag sets for DeleteImageRegistryOptions +func (o *DeleteRegistryOptions) Flags() cliflag.NamedFlagSets { + fss := o.CommonOptions.Flags() + kfs := fss.FlagSet("config") + kfs.StringVar(&o.Kubernetes, "with-kubernetes", o.Kubernetes, fmt.Sprintf("Specify a supported version of kubernetes. default is %s", o.Kubernetes)) + + return fss +} + +// Complete validates and completes the DeleteImageRegistryOptions configuration +func (o *DeleteRegistryOptions) Complete(cmd *cobra.Command, args []string) (*kkcorev1.Playbook, error) { + // Initialize playbook metadata for deleting image registry + playbook := &kkcorev1.Playbook{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "delete-imageregistry-", + Namespace: o.Namespace, + Annotations: map[string]string{ + kkcorev1.BuiltinsProjectAnnotation: "", + }, + }, + } + + // Validate playbook arguments: must have exactly one argument (the playbook) + if len(args) != 1 { + return nil, errors.Errorf("%s\nSee '%s -h' for help and examples", cmd.Use, cmd.CommandPath()) + } + o.Playbook = args[0] + + // Set playbook specification + playbook.Spec = kkcorev1.PlaybookSpec{ + Playbook: o.Playbook, + } + + // Complete common options (e.g., config, inventory) + if err := o.CommonOptions.Complete(playbook); err != nil { + return nil, err + } + + // Complete config specific to delete image registry + return playbook, nil +} diff --git a/cmd/kk/app/options/builtin/init.go b/cmd/kk/app/options/builtin/init.go new file mode 100644 index 000000000..82a7a132a --- /dev/null +++ b/cmd/kk/app/options/builtin/init.go @@ -0,0 +1,159 @@ +//go:build builtin +// +build builtin + +/* +Copyright 2024 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builtin + +import ( + "fmt" + "path/filepath" + + "github.com/cockroachdb/errors" + kkcorev1 "github.com/kubesphere/kubekey/api/core/v1" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + cliflag "k8s.io/component-base/cli/flag" + + "github.com/kubesphere/kubekey/v4/cmd/kk/app/options" + _const "github.com/kubesphere/kubekey/v4/pkg/const" +) + +// ====================================================================================== +// init os +// ====================================================================================== + +// InitOSOptions for NewInitOSOptions +type InitOSOptions struct { + options.CommonOptions +} + +// NewInitOSOptions for newInitOSCommand +func NewInitOSOptions() *InitOSOptions { + // set default value + o := &InitOSOptions{CommonOptions: options.NewCommonOptions()} + o.CommonOptions.GetInventoryFunc = getInventory + + return o +} + +// Flags add to newInitOSCommand +func (o *InitOSOptions) Flags() cliflag.NamedFlagSets { + return o.CommonOptions.Flags() +} + +// Complete options. create Playbook, Config and Inventory +func (o *InitOSOptions) Complete(cmd *cobra.Command, args []string) (*kkcorev1.Playbook, error) { + playbook := &kkcorev1.Playbook{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "init-os-", + Namespace: o.Namespace, + Annotations: map[string]string{ + kkcorev1.BuiltinsProjectAnnotation: "", + }, + }, + } + + // complete playbook. now only support one playbook + if len(args) != 1 { + return nil, errors.Errorf("%s\nSee '%s -h' for help and examples", cmd.Use, cmd.CommandPath()) + } + o.Playbook = args[0] + + playbook.Spec = kkcorev1.PlaybookSpec{ + Playbook: o.Playbook, + } + + if err := o.CommonOptions.Complete(playbook); err != nil { + return nil, err + } + + return playbook, o.completeConfig() +} + +func (o *InitOSOptions) completeConfig() error { + if wd, _, err := unstructured.NestedString(o.CommonOptions.Config.Value(), _const.Workdir); err != nil { + // workdir should set by CommonOptions + return errors.Wrapf(err, "failed to get %q in config", _const.Workdir) + } else { + // set binary dir if not set + if _, _, err := unstructured.NestedString(o.CommonOptions.Config.Value(), _const.BinaryDir); err != nil { + // workdir should set by CommonOptions + if err := unstructured.SetNestedField(o.CommonOptions.Config.Value(), filepath.Join(wd, "kubekey"), _const.BinaryDir); err != nil { + return errors.Wrapf(err, "failed to set %q in config", _const.Workdir) + } + } + } + + return nil +} + +// ====================================================================================== +// init registry +// ====================================================================================== + +// InitRegistryOptions for NewInitRegistryOptions +type InitRegistryOptions struct { + options.CommonOptions + // kubernetes version which the config will install. + Kubernetes string +} + +// NewInitRegistryOptions for newInitRegistryCommand +func NewInitRegistryOptions() *InitRegistryOptions { + // set default value + o := &InitRegistryOptions{CommonOptions: options.NewCommonOptions()} + o.CommonOptions.GetInventoryFunc = getInventory + + return o +} + +// Flags add to newInitRegistryCommand +func (o *InitRegistryOptions) Flags() cliflag.NamedFlagSets { + fss := o.CommonOptions.Flags() + kfs := fss.FlagSet("config") + kfs.StringVar(&o.Kubernetes, "with-kubernetes", o.Kubernetes, fmt.Sprintf("Specify a supported version of kubernetes. default is %s", o.Kubernetes)) + + return fss +} + +// Complete options. create Playbook, Config and Inventory +func (o *InitRegistryOptions) Complete(cmd *cobra.Command, args []string) (*kkcorev1.Playbook, error) { + playbook := &kkcorev1.Playbook{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "init-registry-", + Namespace: o.Namespace, + Annotations: map[string]string{ + kkcorev1.BuiltinsProjectAnnotation: "", + }, + }, + } + + // complete playbook. now only support one playbook + if len(args) != 1 { + return nil, errors.Errorf("%s\nSee '%s -h' for help and examples", cmd.Use, cmd.CommandPath()) + } + o.Playbook = args[0] + + playbook.Spec = kkcorev1.PlaybookSpec{ + Playbook: o.Playbook, + Tags: []string{"image_registry"}, + } + + return playbook, o.CommonOptions.Complete(playbook) +} diff --git a/cmd/kk/app/options/builtin/precheck.go b/cmd/kk/app/options/builtin/precheck.go new file mode 100644 index 000000000..c72900071 --- /dev/null +++ b/cmd/kk/app/options/builtin/precheck.go @@ -0,0 +1,91 @@ +//go:build builtin +// +build builtin + +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builtin + +import ( + "fmt" + + "github.com/cockroachdb/errors" + kkcorev1 "github.com/kubesphere/kubekey/api/core/v1" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + cliflag "k8s.io/component-base/cli/flag" + + "github.com/kubesphere/kubekey/v4/cmd/kk/app/options" +) + +// NewPreCheckOptions for newPreCheckCommand +func NewPreCheckOptions() *PreCheckOptions { + // set default value + o := &PreCheckOptions{ + CommonOptions: options.NewCommonOptions(), + Kubernetes: defaultKubeVersion, + } + o.CommonOptions.GetInventoryFunc = getInventory + + return o +} + +// PreCheckOptions for NewPreCheckOptions +type PreCheckOptions struct { + options.CommonOptions + // kubernetes version which the cluster will install. + Kubernetes string +} + +// Flags add to newPreCheckCommand +func (o *PreCheckOptions) Flags() cliflag.NamedFlagSets { + fss := o.CommonOptions.Flags() + kfs := fss.FlagSet("config") + kfs.StringVar(&o.Kubernetes, "with-kubernetes", o.Kubernetes, fmt.Sprintf("Specify a supported version of kubernetes. default is %s", o.Kubernetes)) + + return fss +} + +// Complete options. create Playbook, Config and Inventory +func (o *PreCheckOptions) Complete(cmd *cobra.Command, args []string) (*kkcorev1.Playbook, error) { + playbook := &kkcorev1.Playbook{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "precheck-", + Namespace: o.Namespace, + Annotations: map[string]string{ + kkcorev1.BuiltinsProjectAnnotation: "", + }, + }, + } + + // complete playbook. now only support one playbook + var tags []string + if len(args) < 1 { + return nil, errors.Errorf("%s\nSee '%s -h' for help and examples", cmd.Use, cmd.CommandPath()) + } else if len(args) == 1 { + o.Playbook = args[0] + } else { + tags = args[:len(args)-1] + o.Playbook = args[len(args)-1] + } + + playbook.Spec = kkcorev1.PlaybookSpec{ + Playbook: o.Playbook, + Tags: tags, + } + + return playbook, o.CommonOptions.Complete(playbook) +} diff --git a/cmd/kk/app/options/option.go b/cmd/kk/app/options/option.go new file mode 100644 index 000000000..ed7a82708 --- /dev/null +++ b/cmd/kk/app/options/option.go @@ -0,0 +1,325 @@ +/* +Copyright 2024 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + + "github.com/cockroachdb/errors" + kkcorev1 "github.com/kubesphere/kubekey/api/core/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/rest" + cliflag "k8s.io/component-base/cli/flag" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + _const "github.com/kubesphere/kubekey/v4/pkg/const" + "github.com/kubesphere/kubekey/v4/pkg/manager" + "github.com/kubesphere/kubekey/v4/pkg/proxy" +) + +// InventoryFunc defines a function type that returns a pointer to a kkcorev1.Inventory and an error. +// It is used to provide a custom way to retrieve or generate an Inventory object. +type InventoryFunc func() (*kkcorev1.Inventory, error) + +// ConfigFunc defines a function type that returns a pointer to a kkcorev1.Config and an error. +// It is used to provide a custom way to retrieve or generate a Config object. +type ConfigFunc func() (*kkcorev1.Config, error) + +// CommonOptions holds the configuration options for executing a playbook. +// It includes paths to various configuration files, runtime settings, and +// debug options. +type CommonOptions struct { + // Playbook specifies the playbook to execute. + Playbook string + // InventoryFile is the path to the host file. + InventoryFile string + // ConfigFile is the path to the configuration file. + ConfigFile string + // Set contains values to set in the configuration. + Set []string + // Workdir is the base directory where the command finds any resources (e.g., project files). + Workdir string + // Artifact is the path to the offline package for kubekey. + Artifact string + // Namespace specifies the namespace for all resources. + Namespace string + + // Config is the kubekey core configuration. + Config *kkcorev1.Config + // Inventory is the kubekey core inventory. + Inventory *kkcorev1.Inventory + GetInventoryFunc InventoryFunc +} + +// NewCommonOptions creates a new CommonOptions object with default values. +// It sets the default namespace, working directory, and initializes the Config and Inventory objects. +func NewCommonOptions() CommonOptions { + o := CommonOptions{ + Namespace: metav1.NamespaceDefault, + } + + // Set the working directory to the current directory joined with "kubekey". + wd, err := os.Getwd() + if err != nil { + klog.ErrorS(err, "get current dir error") + o.Workdir = "/root/kubekey" + } else { + o.Workdir = filepath.Join(wd, "kubekey") + } + + // Initialize the Config object with default API version and kind. + o.Config = &kkcorev1.Config{ + TypeMeta: metav1.TypeMeta{ + APIVersion: kkcorev1.SchemeGroupVersion.String(), + Kind: "Config", + }, + } + + // Initialize the Inventory object with default API version, kind, and name. + o.Inventory = &kkcorev1.Inventory{ + TypeMeta: metav1.TypeMeta{ + APIVersion: kkcorev1.SchemeGroupVersion.String(), + Kind: "Inventory", + }, + ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceDefault, Name: "default"}, + } + + return o +} + +// Run executes the main command logic for the application. +// It sets up the necessary configurations, creates the inventory and playbook +// resources, and then runs the command manager. +func (o *CommonOptions) Run(ctx context.Context, playbook *kkcorev1.Playbook) error { + // create workdir directory,if not exists + if _, err := os.Stat(o.Workdir); err != nil { + if !os.IsNotExist(err) { + return errors.Wrapf(err, "failed to stat local dir %q for playbook %q", o.Workdir, ctrlclient.ObjectKeyFromObject(playbook)) + } + // if dir is not exist, create it. + if err := os.MkdirAll(o.Workdir, os.ModePerm); err != nil { + return errors.Wrapf(err, "failed to create local dir %q for playbook %q", o.Workdir, ctrlclient.ObjectKeyFromObject(playbook)) + } + } + restconfig := &rest.Config{} + if err := proxy.RestConfig(filepath.Join(o.Workdir, _const.RuntimeDir), restconfig); err != nil { + return err + } + client, err := ctrlclient.New(restconfig, ctrlclient.Options{ + Scheme: _const.Scheme, + }) + if err != nil { + return errors.Wrap(err, "failed to runtime-client") + } + // create inventory + if err := client.Create(ctx, o.Inventory); err != nil { + return errors.Wrap(err, "failed to create inventory") + } + // create playbook + if err := client.Create(ctx, playbook); err != nil { + return errors.Wrap(err, "failed to create playbook") + } + + return manager.NewCommandManager(manager.CommandManagerOptions{ + Playbook: playbook, + Config: o.Config, + Inventory: o.Inventory, + Client: client, + }).Run(ctx) +} + +// Flags returns a NamedFlagSets object that contains the command-line flags +// for the CommonOptions. These flags can be used to configure the CommonOptions +// from the command line. +func (o *CommonOptions) Flags() cliflag.NamedFlagSets { + fss := cliflag.NamedFlagSets{} + gfs := fss.FlagSet("generic") + gfs.StringVar(&o.Workdir, "workdir", o.Workdir, "the base Dir for kubekey. Default current dir. ") + gfs.StringVarP(&o.Artifact, "artifact", "a", "", "Path to a KubeKey artifact") + gfs.StringVarP(&o.ConfigFile, "config", "c", o.ConfigFile, "the config file path. support *.yaml ") + gfs.StringArrayVar(&o.Set, "set", o.Set, "set value in config. format --set key=val or --set k1=v1,k2=v2") + gfs.StringVarP(&o.InventoryFile, "inventory", "i", o.InventoryFile, "the host list file path. support *.yaml") + gfs.StringVarP(&o.Namespace, "namespace", "n", o.Namespace, "the namespace which playbook will be executed, all reference resources(playbook, config, inventory, task) should in the same namespace") + + return fss +} + +// Complete finalizes the CommonOptions by setting up the working directory, +// generating the configuration, and completing the inventory reference for the playbook. +func (o *CommonOptions) Complete(playbook *kkcorev1.Playbook) error { + // Ensure the working directory is an absolute path. + if !filepath.IsAbs(o.Workdir) { + wd, err := os.Getwd() + if err != nil { + return errors.Wrap(err, "get current dir error") + } + o.Workdir = filepath.Join(wd, o.Workdir) + } + + if o.ConfigFile != "" { + data, err := os.ReadFile(o.ConfigFile) + if err != nil { + return errors.Wrapf(err, "failed to get config from file %q", o.ConfigFile) + } + if err := yaml.Unmarshal(data, o.Config); err != nil { + return errors.Wrapf(err, "failed to unmarshal config from file %q", o.ConfigFile) + } + } + if o.InventoryFile != "" { + data, err := os.ReadFile(o.InventoryFile) + if err != nil { + return errors.Wrapf(err, "failed to get inventory from file %q", o.InventoryFile) + } + if err := yaml.Unmarshal(data, o.Inventory); err != nil { + return errors.Wrapf(err, "failed to unmarshal inventory from file %q", o.InventoryFile) + } + } else if o.GetInventoryFunc != nil { + inventory, err := o.GetInventoryFunc() + if err != nil { + return err + } + o.Inventory = inventory + } + + // Complete the configuration. + if err := o.completeConfig(); err != nil { + return err + } + playbook.Spec.Config = ptr.Deref(o.Config, kkcorev1.Config{}) + // Complete the inventory reference. + if err := o.completeInventory(o.Inventory); err != nil { + return err + } + playbook.Spec.InventoryRef = &corev1.ObjectReference{ + Kind: o.Inventory.Kind, + Namespace: o.Inventory.Namespace, + Name: o.Inventory.Name, + UID: o.Inventory.UID, + APIVersion: o.Inventory.APIVersion, + ResourceVersion: o.Inventory.ResourceVersion, + } + + return nil +} + +// genConfig generate config by ConfigFile and set value by command args. +func (o *CommonOptions) completeConfig() error { + // set value by command args + if o.Workdir != "" { + if err := unstructured.SetNestedField(o.Config.Value(), o.Workdir, _const.Workdir); err != nil { + return errors.Wrapf(err, "failed to set %q to config", _const.Workdir) + } + } + if o.Artifact != "" { + // override artifact_file in config + if err := unstructured.SetNestedField(o.Config.Value(), o.Artifact, "artifact_file"); err != nil { + return errors.Wrapf(err, "failed to set %q to config", "artifact_file") + } + } + for _, s := range o.Set { + for _, setVal := range strings.Split(unescapeString(s), ",") { + i := strings.Index(setVal, "=") + if i == 0 || i == -1 { + return errors.New("--set value should be k=v") + } + if err := setValue(o.Config, setVal[:i], setVal[i+1:]); err != nil { + return err + } + } + } + + return nil +} + +// genConfig generate config by ConfigFile and set value by command args. +func (o *CommonOptions) completeInventory(inventory *kkcorev1.Inventory) error { + // set value by command args + if o.Namespace != "" { + inventory.Namespace = o.Namespace + } + + return nil +} + +// setValue sets a value in the config based on a key-value pair. +// It supports different value types: +// - JSON objects (starting with '{' and ending with '}') +// - JSON arrays (starting with '[' and ending with ']') +// - Boolean values (true/false, yes/no, y/n - case insensitive) +// - String values (default case) +// The key can contain dots to indicate nested fields. +func setValue(config *kkcorev1.Config, key, val string) error { + switch { + case strings.HasPrefix(val, "{") && strings.HasSuffix(val, "{"): + var value map[string]any + err := json.Unmarshal([]byte(val), &value) + if err != nil { + return errors.Wrapf(err, "failed to unmarshal json object value for \"--set %s\"", key) + } + + return errors.Wrapf(unstructured.SetNestedMap(config.Value(), value, strings.Split(key, ".")...), + "failed to set \"--set %s\" to config", key) + case strings.HasPrefix(val, "[") && strings.HasSuffix(val, "]"): + var value []any + err := json.Unmarshal([]byte(val), &value) + if err != nil { + return errors.Wrapf(err, "failed to unmarshal json array value for \"--set %s\"", key) + } + + return errors.Wrapf(unstructured.SetNestedSlice(config.Value(), value, strings.Split(key, ".")...), + "failed to set \"--set %s\" to config", key) + case strings.EqualFold(val, "TRUE") || strings.EqualFold(val, "YES") || strings.EqualFold(val, "Y"): + return errors.Wrapf(unstructured.SetNestedField(config.Value(), true, strings.Split(key, ".")...), + "failed to set \"--set %s\" to config", key) + case strings.EqualFold(val, "FALSE") || strings.EqualFold(val, "NO") || strings.EqualFold(val, "N"): + return errors.Wrapf(unstructured.SetNestedField(config.Value(), false, strings.Split(key, ".")...), + "failed to set \"--set %s\" to config", key) + default: + return errors.Wrapf(unstructured.SetNestedField(config.Value(), val, strings.Split(key, ".")...), + "failed to set \"--set %s\" to config", key) + } +} + +// unescapeString handles common escape sequences +func unescapeString(s string) string { + replacements := map[string]string{ + `\\`: `\`, + `\"`: `"`, + `\'`: `'`, + `\n`: "\n", + `\r`: "\r", + `\t`: "\t", + `\b`: "\b", + `\f`: "\f", + } + + // Iterate over the replacements map and replace escape sequences in the string + for o, n := range replacements { + s = strings.ReplaceAll(s, o, n) + } + + return s +} diff --git a/cmd/kk/app/options/other.go b/cmd/kk/app/options/other.go new file mode 100644 index 000000000..870d038a8 --- /dev/null +++ b/cmd/kk/app/options/other.go @@ -0,0 +1,165 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "context" + "flag" + "fmt" + "os" + "runtime" + "runtime/pprof" + "strings" + + "github.com/cockroachdb/errors" + "github.com/google/gops/agent" + "github.com/spf13/pflag" + "k8s.io/klog/v2" +) + +// ====================================================================================== +// PROFILING +// ====================================================================================== + +var ( + profileName string + profileOutput string +) + +// AddProfilingFlags to NewRootCommand +func AddProfilingFlags(flags *pflag.FlagSet) { + flags.StringVar(&profileName, "profile", "none", "Name of profile to capture. One of (none|cpu|heap|goroutine|threadcreate|block|mutex)") + flags.StringVar(&profileOutput, "profile-output", "profile.pprof", "Name of the file to write the profile to") +} + +// InitProfiling for profileName +func InitProfiling(ctx context.Context) error { + var ( + f *os.File + err error + ) + + switch profileName { + case "none": + return nil + case "cpu": + f, err = os.Create(profileOutput) + if err != nil { + return errors.Wrap(err, "failed to create cpu profile") + } + + err = pprof.StartCPUProfile(f) + if err != nil { + return errors.Wrap(err, "failed to start cpu profile") + } + // Block and mutex profiles need a call to Set{Block,Mutex}ProfileRate to + // output anything. We choose to sample all events. + case "block": + runtime.SetBlockProfileRate(1) + case "mutex": + runtime.SetMutexProfileFraction(1) + default: + // Check the profile name is valid. + if profile := pprof.Lookup(profileName); profile == nil { + return errors.Errorf("unknown profile '%s'", profileName) + } + } + + // If the command is interrupted before the end (ctrl-c), flush the + // profiling files + go func() { + <-ctx.Done() + if err := f.Close(); err != nil { + fmt.Printf("failed to close file. file: %v. error: %v \n", profileOutput, err) + } + if err := FlushProfiling(); err != nil { + fmt.Printf("failed to FlushProfiling. file: %v. error: %v \n", profileOutput, err) + } + }() + + return nil +} + +// FlushProfiling to local file +func FlushProfiling() error { + switch profileName { + case "none": + return nil + case "cpu": + pprof.StopCPUProfile() + case "heap": + runtime.GC() + + fallthrough + default: + profile := pprof.Lookup(profileName) + if profile == nil { + return nil + } + + f, err := os.Create(profileOutput) + if err != nil { + return errors.Wrapf(err, "failed to create profile %s", profileName) + } + defer f.Close() + + if err := profile.WriteTo(f, 0); err != nil { + return errors.Wrapf(err, "failed to write profile %s", profileName) + } + } + + return nil +} + +// ====================================================================================== +// GOPS +// ====================================================================================== + +var gops bool + +// AddGOPSFlags to NewRootCommand +func AddGOPSFlags(flags *pflag.FlagSet) { + flags.BoolVar(&gops, "gops", false, "Whether to enable gops or not. When enabled this option, "+ + "controller-manager will listen on a random port on 127.0.0.1, then you can use the gops tool to list and diagnose the controller-manager currently running.") +} + +// InitGOPS if gops is true +func InitGOPS() error { + if gops { + // Add agent to report additional information such as the current stack trace, Go version, memory stats, etc. + // Bind to a random port on address 127.0.0.1 + if err := agent.Listen(agent.Options{}); err != nil { + return errors.Wrap(err, "failed to listen gops agent") + } + } + + return nil +} + +// ====================================================================================== +// KLOG +// ====================================================================================== + +// AddKlogFlags to NewRootCommand +func AddKlogFlags(fs *pflag.FlagSet) { + local := flag.NewFlagSet("klog", flag.ExitOnError) + klog.InitFlags(local) + local.VisitAll(func(fl *flag.Flag) { + fl.Name = strings.Replace(fl.Name, "_", "-", -1) + fs.AddGoFlag(fl) + }) +} diff --git a/cmd/kk/app/options/playbook.go b/cmd/kk/app/options/playbook.go new file mode 100644 index 000000000..a364aada0 --- /dev/null +++ b/cmd/kk/app/options/playbook.go @@ -0,0 +1,29 @@ +package options + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + cliflag "k8s.io/component-base/cli/flag" +) + +// PlaybookOptions for NewPlaybookOptions +type PlaybookOptions struct { + Name string + Namespace string +} + +// NewPlaybookOptions for newPlaybookCommand +func NewPlaybookOptions() *PlaybookOptions { + return &PlaybookOptions{ + Namespace: metav1.NamespaceDefault, + } +} + +// Flags add to newPlaybookCommand +func (o *PlaybookOptions) Flags() cliflag.NamedFlagSets { + fss := cliflag.NamedFlagSets{} + pfs := fss.FlagSet("playbook flags") + pfs.StringVar(&o.Name, "name", o.Name, "name of playbook") + pfs.StringVarP(&o.Namespace, "namespace", "n", o.Namespace, "namespace of playbook") + + return fss +} diff --git a/cmd/kk/app/options/run.go b/cmd/kk/app/options/run.go new file mode 100644 index 000000000..68e398938 --- /dev/null +++ b/cmd/kk/app/options/run.go @@ -0,0 +1,125 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "os" + + "github.com/cockroachdb/errors" + kkcorev1 "github.com/kubesphere/kubekey/api/core/v1" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/yaml" + cliflag "k8s.io/component-base/cli/flag" +) + +// KubeKeyRunOptions for NewKubeKeyRunOptions +type KubeKeyRunOptions struct { + CommonOptions + // ProjectAddr is the storage for executable packages (in Ansible format). + // When starting with http or https, it will be obtained from a Git repository. + // When starting with file path, it will be obtained from the local path. + ProjectAddr string + // ProjectName is the name of project. it will store to project dir use this name. + // If empty generate from ProjectAddr + ProjectName string + // ProjectBranch is the git branch of the git Addr. + ProjectBranch string + // ProjectTag if the git tag of the git Addr. + ProjectTag string + // ProjectInsecureSkipTLS skip tls or not when git addr is https. + ProjectInsecureSkipTLS bool + // ProjectToken to clone and pull git project + ProjectToken string + // Tags is the tags of playbook which to execute + Tags []string + // SkipTags is the tags of playbook which skip execute + SkipTags []string +} + +// NewKubeKeyRunOptions for newRunCommand +func NewKubeKeyRunOptions() *KubeKeyRunOptions { + // add default values + o := &KubeKeyRunOptions{ + CommonOptions: NewCommonOptions(), + } + + return o +} + +// Flags add to newRunCommand +func (o *KubeKeyRunOptions) Flags() cliflag.NamedFlagSets { + fss := o.CommonOptions.Flags() + gitfs := fss.FlagSet("project") + gitfs.StringVar(&o.ProjectAddr, "project-addr", o.ProjectAddr, "the storage for executable packages (in Ansible format)."+ + " When starting with http or https, it will be obtained from a Git repository."+ + "When starting with file path, it will be obtained from the local path.") + gitfs.StringVar(&o.ProjectBranch, "project-branch", o.ProjectBranch, "the git branch of the remote Addr") + gitfs.StringVar(&o.ProjectTag, "project-tag", o.ProjectTag, "the git tag of the remote Addr") + gitfs.BoolVar(&o.ProjectInsecureSkipTLS, "project-insecure-skip-tls", o.ProjectInsecureSkipTLS, "skip tls or not when git addr is https.") + gitfs.StringVar(&o.ProjectToken, "project-token", o.ProjectToken, "the token for private project.") + + tfs := fss.FlagSet("tags") + tfs.StringArrayVar(&o.Tags, "tags", o.Tags, "the tags of playbook which to execute") + tfs.StringArrayVar(&o.SkipTags, "skip-tags", o.SkipTags, "the tags of playbook which skip execute") + + return fss +} + +// Complete options. create Playbook, Config and Inventory +func (o *KubeKeyRunOptions) Complete(cmd *cobra.Command, args []string) (*kkcorev1.Playbook, error) { + playbook := &kkcorev1.Playbook{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "run-", + Namespace: o.Namespace, + Annotations: make(map[string]string), + }, + } + // complete playbook. now only support one playbook + if len(args) != 1 { + return nil, errors.Errorf("%s\nSee '%s -h' for help and examples", cmd.Use, cmd.CommandPath()) + } + o.Playbook = args[0] + + playbook.Spec = kkcorev1.PlaybookSpec{ + Project: kkcorev1.PlaybookProject{ + Addr: o.ProjectAddr, + Name: o.ProjectName, + Branch: o.ProjectBranch, + Tag: o.ProjectTag, + InsecureSkipTLS: o.ProjectInsecureSkipTLS, + Token: o.ProjectToken, + }, + Playbook: o.Playbook, + Tags: o.Tags, + SkipTags: o.SkipTags, + } + + if o.InventoryFile != "" { + data, err := os.ReadFile(o.InventoryFile) + if err != nil { + return nil, errors.Wrapf(err, "failed to get inventory for inventoryFile: %q", &o.InventoryFile) + } + + err = yaml.Unmarshal(data, o.Inventory) + if err != nil { + return nil, errors.Wrapf(yaml.Unmarshal(data, o.Inventory), "failed to unmarshal inventoryFile %s", o.InventoryFile) + } + } + + return playbook, o.CommonOptions.Complete(playbook) +} diff --git a/cmd/kk/app/options/web.go b/cmd/kk/app/options/web.go new file mode 100644 index 000000000..a3d5b14b3 --- /dev/null +++ b/cmd/kk/app/options/web.go @@ -0,0 +1,53 @@ +package options + +import ( + "fmt" + "os" + "path/filepath" + + cliflag "k8s.io/component-base/cli/flag" + "k8s.io/klog/v2" +) + +// defaultPort defines the default port number for the web server +const ( + defaultPort = 80 +) + +// KubeKeyWebOptions contains configuration options for the KubeKey web server +type KubeKeyWebOptions struct { + Port int // Port specifies the port number for the web server + Workdir string // Workdir specifies the base directory for KubeKey + + SchemaPath string + UIPath string +} + +// NewKubeKeyWebOptions creates and returns a new KubeKeyWebOptions instance with default values +func NewKubeKeyWebOptions() *KubeKeyWebOptions { + o := &KubeKeyWebOptions{ + Port: defaultPort, + } + // Set the working directory to the current directory joined with "kubekey". + wd, err := os.Getwd() + if err != nil { + klog.ErrorS(err, "get current dir error") + o.Workdir = "/root/kubekey" + } else { + o.Workdir = filepath.Join(wd, "kubekey") + } + + return o +} + +// Flags returns a NamedFlagSets object containing command-line flags for configuring the web server +func (o *KubeKeyWebOptions) Flags() cliflag.NamedFlagSets { + fss := cliflag.NamedFlagSets{} + wfs := fss.FlagSet("web flags") + wfs.IntVar(&o.Port, "port", o.Port, fmt.Sprintf("the server port of kubekey web default is: %d", o.Port)) + wfs.StringVar(&o.Workdir, "workdir", o.Workdir, "the base Dir for kubekey. Default current dir. ") + wfs.StringVar(&o.SchemaPath, "schema-path", o.SchemaPath, "the json schema dir path to render web ui.") + wfs.StringVar(&o.UIPath, "ui-path", o.SchemaPath, "the web ui package path.") + + return fss +} diff --git a/cmd/kk/app/playbook.go b/cmd/kk/app/playbook.go new file mode 100644 index 000000000..9884a0422 --- /dev/null +++ b/cmd/kk/app/playbook.go @@ -0,0 +1,81 @@ +package app + +import ( + "path/filepath" + + "github.com/cockroachdb/errors" + kkcorev1 "github.com/kubesphere/kubekey/api/core/v1" + "github.com/spf13/cobra" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kubesphere/kubekey/v4/cmd/kk/app/options" + _const "github.com/kubesphere/kubekey/v4/pkg/const" + "github.com/kubesphere/kubekey/v4/pkg/manager" + "github.com/kubesphere/kubekey/v4/pkg/proxy" +) + +func newPlaybookCommand() *cobra.Command { + o := options.NewPlaybookOptions() + + cmd := &cobra.Command{ + Use: "playbook", + Short: "Executor a playbook in kubernetes", + RunE: func(cmd *cobra.Command, _ []string) error { + restconfig, err := ctrl.GetConfig() + if err != nil { + return errors.Wrap(err, "failed to get restconfig") + } + kubeclient, err := ctrlclient.New(restconfig, ctrlclient.Options{ + Scheme: _const.Scheme, + }) + if err != nil { + return errors.Wrap(err, "failed to create client") + } + // get playbook + playbook := &kkcorev1.Playbook{} + if err := kubeclient.Get(cmd.Context(), ctrlclient.ObjectKey{ + Name: o.Name, + Namespace: o.Namespace, + }, playbook); err != nil { + return errors.Wrap(err, "failed to get playbook") + } + if playbook.Status.Phase != kkcorev1.PlaybookPhaseRunning { + klog.InfoS("playbook is not running, skip", "playbook", ctrlclient.ObjectKeyFromObject(playbook)) + + return nil + } + // get inventory + inventory := new(kkcorev1.Inventory) + if err := kubeclient.Get(cmd.Context(), ctrlclient.ObjectKey{ + Name: playbook.Spec.InventoryRef.Name, + Namespace: playbook.Spec.InventoryRef.Namespace, + }, inventory); err != nil { + return errors.Wrap(err, "failed to get inventory") + } + if err := proxy.RestConfig(filepath.Join(_const.GetWorkdirFromConfig(playbook.Spec.Config), _const.RuntimeDir), restconfig); err != nil { + return err + } + // use proxy client to store task. + proxyclient, err := ctrlclient.New(restconfig, ctrlclient.Options{ + Scheme: _const.Scheme, + }) + if err != nil { + return errors.Wrap(err, "failed to create client") + } + + return manager.NewCommandManager(manager.CommandManagerOptions{ + Playbook: playbook, + Inventory: inventory, + Client: proxyclient, + }).Run(cmd.Context()) + }, + } + fs := cmd.Flags() + for _, f := range o.Flags().FlagSets { + fs.AddFlagSet(f) + } + + return cmd +} diff --git a/cmd/kk/app/root.go b/cmd/kk/app/root.go new file mode 100644 index 000000000..8fe9da1ae --- /dev/null +++ b/cmd/kk/app/root.go @@ -0,0 +1,60 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "github.com/spf13/cobra" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" + + "github.com/kubesphere/kubekey/v4/cmd/kk/app/options" +) + +var internalCommand = make([]*cobra.Command, 0) + +// NewRootCommand console command. +func NewRootCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "kk", + Long: "kubekey is a daemon that execute command in a node", + PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { + if err := options.InitGOPS(); err != nil { + return err + } + + return options.InitProfiling(cmd.Context()) + }, + PersistentPostRunE: func(*cobra.Command, []string) error { + return options.FlushProfiling() + }, + SilenceUsage: true, + } + cmd.SetContext(signals.SetupSignalHandler()) + // add common flag + flags := cmd.PersistentFlags() + options.AddProfilingFlags(flags) + options.AddKlogFlags(flags) + options.AddGOPSFlags(flags) + // add children command + cmd.AddCommand(newRunCommand()) + cmd.AddCommand(newPlaybookCommand()) + cmd.AddCommand(newVersionCommand()) + cmd.AddCommand(newWebCommand()) + // internal command + cmd.AddCommand(internalCommand...) + + return cmd +} diff --git a/cmd/kk/app/run.go b/cmd/kk/app/run.go new file mode 100644 index 000000000..5b1920eed --- /dev/null +++ b/cmd/kk/app/run.go @@ -0,0 +1,45 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "github.com/spf13/cobra" + + "github.com/kubesphere/kubekey/v4/cmd/kk/app/options" +) + +func newRunCommand() *cobra.Command { + o := options.NewKubeKeyRunOptions() + + cmd := &cobra.Command{ + Use: "run [playbook]", + Short: "run a playbook by playbook file. the file source can be git or local", + RunE: func(cmd *cobra.Command, args []string) error { + playbook, err := o.Complete(cmd, args) + if err != nil { + return err + } + + return o.CommonOptions.Run(cmd.Context(), playbook) + }, + } + for _, f := range o.Flags().FlagSets { + cmd.Flags().AddFlagSet(f) + } + + return cmd +} diff --git a/cmd/kk/app/version.go b/cmd/kk/app/version.go new file mode 100644 index 000000000..84a80c450 --- /dev/null +++ b/cmd/kk/app/version.go @@ -0,0 +1,33 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "github.com/spf13/cobra" + + "github.com/kubesphere/kubekey/v4/version" +) + +func newVersionCommand() *cobra.Command { + return &cobra.Command{ + Use: "version", + Short: "Print the version of KubeSphere controller-manager", + Run: func(cmd *cobra.Command, _ []string) { + cmd.Println(version.Get()) + }, + } +} diff --git a/cmd/kk/app/web.go b/cmd/kk/app/web.go new file mode 100644 index 000000000..a7d2b4cc9 --- /dev/null +++ b/cmd/kk/app/web.go @@ -0,0 +1,56 @@ +package app + +import ( + "path/filepath" + + "github.com/spf13/cobra" + "k8s.io/client-go/rest" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kubesphere/kubekey/v4/cmd/kk/app/options" + _const "github.com/kubesphere/kubekey/v4/pkg/const" + "github.com/kubesphere/kubekey/v4/pkg/manager" + "github.com/kubesphere/kubekey/v4/pkg/proxy" +) + +// newWebCommand creates a new cobra command for starting the KubeKey web server +// It initializes the web server with the provided configuration options and starts +// the HTTP server with web UI interface +func newWebCommand() *cobra.Command { + o := options.NewKubeKeyWebOptions() + + cmd := &cobra.Command{ + Use: "web", + Short: "start a http server with web UI.", + RunE: func(cmd *cobra.Command, args []string) error { + // Initialize REST config for Kubernetes client + restconfig := &rest.Config{} + if err := proxy.RestConfig(filepath.Join(o.Workdir, _const.RuntimeDir), restconfig); err != nil { + return err + } + + // Create Kubernetes client with the REST config + client, err := ctrlclient.New(restconfig, ctrlclient.Options{ + Scheme: _const.Scheme, + }) + if err != nil { + return err + } + + // Initialize and run the web manager with provided options + return manager.NewWebManager(manager.WebManagerOptions{ + Workdir: o.Workdir, + Port: o.Port, + SchemaPath: o.SchemaPath, + UIPath: o.UIPath, + Client: client, + Config: restconfig, + }).Run(cmd.Context()) + }, + } + for _, f := range o.Flags().FlagSets { + cmd.Flags().AddFlagSet(f) + } + + return cmd +} diff --git a/cmd/kk/cmd/add/add.go b/cmd/kk/cmd/add/add.go deleted file mode 100644 index f4d0f4d4a..000000000 --- a/cmd/kk/cmd/add/add.go +++ /dev/null @@ -1,47 +0,0 @@ -/* - Copyright 2020 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package add - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" -) - -type AddOptions struct { - CommonOptions *options.CommonOptions -} - -func NewAddOptions() *AddOptions { - return &AddOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdAdd creates a new add command -func NewCmdAdd() *cobra.Command { - o := NewAddOptions() - cmd := &cobra.Command{ - Use: "add", - Short: "Add nodes to kubernetes cluster", - } - - o.CommonOptions.AddCommonFlag(cmd) - - cmd.AddCommand(NewCmdAddNodes()) - return cmd -} diff --git a/cmd/kk/cmd/add/add_nodes.go b/cmd/kk/cmd/add/add_nodes.go deleted file mode 100644 index 1caee4722..000000000 --- a/cmd/kk/cmd/add/add_nodes.go +++ /dev/null @@ -1,92 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package add - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/pipelines" -) - -type AddNodesOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - SkipPullImages bool - ContainerManager string - DownloadCmd string - Artifact string - InstallPackages bool -} - -func NewAddNodesOptions() *AddNodesOptions { - return &AddNodesOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdAddNodes creates a new add nodes command -func NewCmdAddNodes() *cobra.Command { - o := NewAddNodesOptions() - cmd := &cobra.Command{ - Use: "nodes", - Short: "Add nodes to the cluster according to the new nodes information from the specified configuration file", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(cmd, args)) - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *AddNodesOptions) Complete(_ *cobra.Command, _ []string) error { - if o.Artifact == "" { - o.InstallPackages = false - } - return nil -} - -func (o *AddNodesOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KsEnable: false, - Debug: o.CommonOptions.Verbose, - IgnoreErr: o.CommonOptions.IgnoreErr, - SkipConfirmCheck: o.CommonOptions.SkipConfirmCheck, - SkipPullImages: o.SkipPullImages, - ContainerManager: o.ContainerManager, - Artifact: o.Artifact, - InstallPackages: o.InstallPackages, - Namespace: o.CommonOptions.Namespace, - } - return pipelines.AddNodes(arg, o.DownloadCmd) -} - -func (o *AddNodesOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().BoolVarP(&o.SkipPullImages, "skip-pull-images", "", false, "Skip pre pull images") - cmd.Flags().StringVarP(&o.ContainerManager, "container-manager", "", "docker", "Container manager: docker, crio, containerd and isula.") - cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s", - `The user defined command to download the necessary binary files. The first param '%s' is output path, the second param '%s', is the URL`) - cmd.Flags().StringVarP(&o.Artifact, "artifact", "a", "", "Path to a KubeKey artifact") - cmd.Flags().BoolVarP(&o.InstallPackages, "with-packages", "", false, "install operation system packages by artifact") -} diff --git a/cmd/kk/cmd/alpha/alpha.go b/cmd/kk/cmd/alpha/alpha.go deleted file mode 100644 index 44fb0bea4..000000000 --- a/cmd/kk/cmd/alpha/alpha.go +++ /dev/null @@ -1,36 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package alpha - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/alpha/cri" -) - -// NewAlphaCmd create a new Alpha command -func NewAlphaCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "alpha", - Short: "Commands for features in alpha", - } - - cmd.AddCommand(NewCmdCreate()) - cmd.AddCommand(NewCmdUpgrade()) - cmd.AddCommand(cri.NewCmdCri()) - return cmd -} diff --git a/cmd/kk/cmd/alpha/create.go b/cmd/kk/cmd/alpha/create.go deleted file mode 100644 index c3ee724dc..000000000 --- a/cmd/kk/cmd/alpha/create.go +++ /dev/null @@ -1,47 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package alpha - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/create/phase" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" -) - -type CreateOptions struct { - CommonOptions *options.CommonOptions -} - -func NewCreateOptions() *CreateOptions { - return &CreateOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCreate creates a new create command -func NewCmdCreate() *cobra.Command { - o := NewCreateOptions() - cmd := &cobra.Command{ - Use: "create", - Short: "Create a cluster by phase cmds for testing", - } - - o.CommonOptions.AddCommonFlag(cmd) - cmd.AddCommand(phase.NewPhaseCommand()) - return cmd -} diff --git a/cmd/kk/cmd/alpha/cri/cri.go b/cmd/kk/cmd/alpha/cri/cri.go deleted file mode 100644 index a9b51c4dd..000000000 --- a/cmd/kk/cmd/alpha/cri/cri.go +++ /dev/null @@ -1,46 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cri - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" -) - -type MigrateOptions struct { - CommonOptions *options.CommonOptions -} - -func NewMigrateOptions() *MigrateOptions { - return &MigrateOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCri creates a new Migrate command -func NewCmdCri() *cobra.Command { - o := NewMigrateOptions() - cmd := &cobra.Command{ - Use: "cri", - Short: "cri", - } - - o.CommonOptions.AddCommonFlag(cmd) - cmd.AddCommand(NewCmdMigrateCri()) - return cmd -} diff --git a/cmd/kk/cmd/alpha/cri/migrate_cri.go b/cmd/kk/cmd/alpha/cri/migrate_cri.go deleted file mode 100644 index c11ec145c..000000000 --- a/cmd/kk/cmd/alpha/cri/migrate_cri.go +++ /dev/null @@ -1,104 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cri - -import ( - "github.com/pkg/errors" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/pipelines" -) - -type MigrateCriOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - Kubernetes string - EnableKubeSphere bool - KubeSphere string - DownloadCmd string - Artifact string - Type string - Role string -} - -func NewMigrateCriOptions() *MigrateCriOptions { - return &MigrateCriOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdMigrateCri creates a new delete cluster command -func NewCmdMigrateCri() *cobra.Command { - o := NewMigrateCriOptions() - cmd := &cobra.Command{ - Use: "migrate", - Short: "Migrate a container", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Validate()) - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *MigrateCriOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - Debug: o.CommonOptions.Verbose, - KubernetesVersion: o.Kubernetes, - Type: o.Type, - Role: o.Role, - } - return pipelines.MigrateCri(arg, o.DownloadCmd) -} - -func (o *MigrateCriOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.Role, "role", "", "", "Role groups for migrating. Support: master, worker, all.") - cmd.Flags().StringVarP(&o.Type, "type", "", "", "Type of target CRI. Support: docker, containerd.") - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") - cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s", - `The user defined command to download the necessary binary files. The first param '%s' is output path, the second param '%s', is the URL`) - cmd.Flags().StringVarP(&o.Artifact, "artifact", "a", "", "Path to a KubeKey artifact") -} - -func (o *MigrateCriOptions) Validate() error { - if o.Role == "" { - return errors.New("node Role can not be empty") - } - if o.Role != common.Worker && o.Role != common.Master && o.Role != "all" { - - return errors.Errorf("node Role is invalid: %s", o.Role) - } - if o.Type == "" { - return errors.New("cri Type can not be empty") - } - if o.Type != common.Docker && o.Type != common.Containerd { - return errors.Errorf("cri Type is invalid: %s", o.Type) - } - if o.ClusterCfgFile == "" { - return errors.New("configuration file can not be empty") - } - return nil -} diff --git a/cmd/kk/cmd/alpha/upgrade.go b/cmd/kk/cmd/alpha/upgrade.go deleted file mode 100644 index 09d739ca9..000000000 --- a/cmd/kk/cmd/alpha/upgrade.go +++ /dev/null @@ -1,46 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package alpha - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/upgrade/phase" -) - -type UpgradeOptions struct { - CommonOptions *options.CommonOptions -} - -func NewUpgradeOptions() *UpgradeOptions { - return &UpgradeOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdUpgrade creates a new upgrade command -func NewCmdUpgrade() *cobra.Command { - o := NewUpgradeOptions() - cmd := &cobra.Command{ - Use: "upgrade", - Short: "Upgrade your cluster by phase cmd for testing", - } - o.CommonOptions.AddCommonFlag(cmd) - cmd.AddCommand(phase.NewPhaseCommand()) - return cmd -} diff --git a/cmd/kk/cmd/artifact/artifact.go b/cmd/kk/cmd/artifact/artifact.go deleted file mode 100755 index 8f046c984..000000000 --- a/cmd/kk/cmd/artifact/artifact.go +++ /dev/null @@ -1,50 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package artifact - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/artifact/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" -) - -type ArtifactOptions struct { - CommonOptions *options.CommonOptions -} - -func NewArtifactOptions() *ArtifactOptions { - return &ArtifactOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdArtifact creates a new cobra.Command for `kubekey artifact` -func NewCmdArtifact() *cobra.Command { - o := NewArtifactOptions() - cmd := &cobra.Command{ - Use: "artifact", - Short: "Manage a KubeKey offline installation package", - } - - o.CommonOptions.AddCommonFlag(cmd) - - cmd.AddCommand(NewCmdArtifactExport()) - cmd.AddCommand(images.NewCmdArtifactImages()) - cmd.AddCommand(NewCmdArtifactImport()) - return cmd -} diff --git a/cmd/kk/cmd/artifact/export.go b/cmd/kk/cmd/artifact/export.go deleted file mode 100644 index b5c2b822f..000000000 --- a/cmd/kk/cmd/artifact/export.go +++ /dev/null @@ -1,94 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package artifact - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/pipelines" -) - -type ArtifactExportOptions struct { - CommonOptions *options.CommonOptions - - ManifestFile string - Output string - CriSocket string - DownloadCmd string -} - -func NewArtifactExportOptions() *ArtifactExportOptions { - return &ArtifactExportOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdArtifactExport creates a new `kubekey artifact export` command -func NewCmdArtifactExport() *cobra.Command { - o := NewArtifactExportOptions() - cmd := &cobra.Command{ - Use: "export", - Short: "Export a KubeKey offline installation package", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(cmd, args)) - util.CheckErr(o.Validate(args)) - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *ArtifactExportOptions) Complete(_ *cobra.Command, _ []string) error { - if o.Output == "" { - o.Output = "kubekey-artifact.tar.gz" - } - return nil -} - -func (o *ArtifactExportOptions) Validate(_ []string) error { - if o.ManifestFile == "" { - return fmt.Errorf("--manifest can not be an empty string") - } - return nil -} - -func (o *ArtifactExportOptions) Run() error { - arg := common.ArtifactArgument{ - ManifestFile: o.ManifestFile, - Output: o.Output, - CriSocket: o.CriSocket, - Debug: o.CommonOptions.Verbose, - IgnoreErr: o.CommonOptions.IgnoreErr, - } - - return pipelines.ArtifactExport(arg, o.DownloadCmd) -} - -func (o *ArtifactExportOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ManifestFile, "manifest", "m", "", "Path to a manifest file") - cmd.Flags().StringVarP(&o.Output, "output", "o", "", "Path to a output path") - cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s", - `The user defined command to download the necessary binary files. The first param '%s' is output path, the second param '%s', is the URL`) -} diff --git a/cmd/kk/cmd/artifact/images/images.go b/cmd/kk/cmd/artifact/images/images.go deleted file mode 100644 index 0d3ae5ebe..000000000 --- a/cmd/kk/cmd/artifact/images/images.go +++ /dev/null @@ -1,48 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package images - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" -) - -type ArtifactImagesOptions struct { - CommonOptions *options.CommonOptions -} - -func NewArtifactImagesOptions() *ArtifactImagesOptions { - return &ArtifactImagesOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdArtifactImages creates a new `kubekey artifact image` command -func NewCmdArtifactImages() *cobra.Command { - o := NewArtifactImagesOptions() - cmd := &cobra.Command{ - Use: "images", - Aliases: []string{"image", "i"}, - Short: "manage KubeKey artifact image", - } - - o.CommonOptions.AddCommonFlag(cmd) - cmd.AddCommand(NewCmdArtifactImagesPush()) - - return cmd -} diff --git a/cmd/kk/cmd/artifact/images/push.go b/cmd/kk/cmd/artifact/images/push.go deleted file mode 100644 index d337bdd14..000000000 --- a/cmd/kk/cmd/artifact/images/push.go +++ /dev/null @@ -1,140 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package images - -import ( - "os" - "path/filepath" - - "github.com/pkg/errors" - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/artifact" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/filesystem" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" -) - -type ArtifactImagesPushOptions struct { - CommonOptions *options.CommonOptions - - ImageDirPath string - Artifact string - ClusterCfgFile string -} - -func NewArtifactImagesPushOptions() *ArtifactImagesPushOptions { - return &ArtifactImagesPushOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdArtifactImagesPush creates a new `kubekey artifacts images push` command -func NewCmdArtifactImagesPush() *cobra.Command { - o := NewArtifactImagesPushOptions() - cmd := &cobra.Command{ - Use: "push", - Short: "push images to a registry from an artifact", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(cmd, args)) - util.CheckErr(o.Validate(args)) - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *ArtifactImagesPushOptions) Complete(_ *cobra.Command, _ []string) error { - if o.ImageDirPath == "" && o.Artifact == "" { - currentDir, err := filepath.Abs(filepath.Dir(os.Args[0])) - if err != nil { - return errors.Wrap(err, "failed to get current directory") - } - o.ImageDirPath = filepath.Join(currentDir, "kubekey", "images") - } - - return nil -} - -func (o *ArtifactImagesPushOptions) Validate(_ []string) error { - if o.ClusterCfgFile == "" { - return errors.New("kubekey config file is required") - } - if o.ImageDirPath != "" && o.Artifact != "" { - return errors.New("only one of --image-dir or --artifact can be specified") - } - return nil -} - -func (o *ArtifactImagesPushOptions) Run() error { - arg := common.Argument{ - ImagesDir: o.ImageDirPath, - Artifact: o.Artifact, - FilePath: o.ClusterCfgFile, - Debug: o.CommonOptions.Verbose, - IgnoreErr: o.CommonOptions.IgnoreErr, - } - return runPush(arg) -} - -func (o *ArtifactImagesPushOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ImageDirPath, "images-dir", "", "", "Path to a KubeKey artifact images directory") - cmd.Flags().StringVarP(&o.Artifact, "artifact", "a", "", "Path to a KubeKey artifact") - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") -} - -func runPush(arg common.Argument) error { - runtime, err := common.NewKubeRuntime(common.File, arg) - if err != nil { - return err - } - - if err := newImagesPushPipeline(runtime); err != nil { - return err - } - - return nil -} - -func newImagesPushPipeline(runtime *common.KubeRuntime) error { - noArtifact := runtime.Arg.Artifact == "" - - m := []module.Module{ - &artifact.UnArchiveModule{Skip: noArtifact}, - &images.CopyImagesToRegistryModule{ImagePath: runtime.Arg.ImagesDir}, - &filesystem.ChownWorkDirModule{}, - } - - p := pipeline.Pipeline{ - Name: "ArtifactImagesPushPipeline", - Modules: m, - Runtime: runtime, - } - - if err := p.Start(); err != nil { - return err - } - - return nil -} diff --git a/cmd/kk/cmd/artifact/import.go b/cmd/kk/cmd/artifact/import.go deleted file mode 100755 index 8707a99d5..000000000 --- a/cmd/kk/cmd/artifact/import.go +++ /dev/null @@ -1,75 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package artifact - -import ( - "errors" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/artifact" -) - -type ArtifactImportOptions struct { - CommonOptions *options.CommonOptions - Artifact string -} - -func NewArtifactImportOptions() *ArtifactImportOptions { - return &ArtifactImportOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdArtifactImport creates a new artifact import command -func NewCmdArtifactImport() *cobra.Command { - o := NewArtifactImportOptions() - cmd := &cobra.Command{ - Use: "import", - Short: "Import a KubeKey offline installation package", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Validate(args)) - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *ArtifactImportOptions) Run() error { - arg := common.Argument{ - Debug: o.CommonOptions.Verbose, - Artifact: o.Artifact, - } - return artifact.ArtifactImport(arg) -} - -func (o *ArtifactImportOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.Artifact, "artifact", "a", "", "Path to a artifact gzip") -} - -func (o *ArtifactImportOptions) Validate(_ []string) error { - if o.Artifact == "" { - return errors.New("artifact path can not be empty") - } - return nil -} diff --git a/cmd/kk/cmd/cert/cert.go b/cmd/kk/cmd/cert/cert.go deleted file mode 100644 index 89c21785e..000000000 --- a/cmd/kk/cmd/cert/cert.go +++ /dev/null @@ -1,48 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cert - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" -) - -type CertOptions struct { - CommonOptions *options.CommonOptions -} - -func NewCertsOptions() *CertOptions { - return &CertOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCerts creates a new cert command -func NewCmdCerts() *cobra.Command { - o := NewCertsOptions() - cmd := &cobra.Command{ - Use: "certs", - Short: "cluster certs", - } - - o.CommonOptions.AddCommonFlag(cmd) - - cmd.AddCommand(NewCmdCertList()) - cmd.AddCommand(NewCmdCertRenew()) - return cmd -} diff --git a/cmd/kk/cmd/cert/list_cert.go b/cmd/kk/cmd/cert/list_cert.go deleted file mode 100644 index 79b58fd2b..000000000 --- a/cmd/kk/cmd/cert/list_cert.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cert - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/pipelines" -) - -type CertListOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string -} - -func NewCertListOptions() *CertListOptions { - return &CertListOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCertList creates a new cert list command -func NewCmdCertList() *cobra.Command { - o := NewCertListOptions() - cmd := &cobra.Command{ - Use: "check-expiration", - Short: "Check certificates expiration for a Kubernetes cluster", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *CertListOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - Debug: o.CommonOptions.Verbose, - } - return pipelines.CheckCerts(arg) -} - -func (o *CertListOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") -} diff --git a/cmd/kk/cmd/cert/renew_cert.go b/cmd/kk/cmd/cert/renew_cert.go deleted file mode 100644 index f3dcbd82c..000000000 --- a/cmd/kk/cmd/cert/renew_cert.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cert - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/pipelines" -) - -type CertRenewOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string -} - -func NewCertRenewOptions() *CertRenewOptions { - return &CertRenewOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCertRenew creates a new cert renew command -func NewCmdCertRenew() *cobra.Command { - o := NewCertRenewOptions() - cmd := &cobra.Command{ - Use: "renew", - Short: "renew a cluster certs", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *CertRenewOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - Debug: o.CommonOptions.Verbose, - } - return pipelines.RenewCerts(arg) -} - -func (o *CertRenewOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") -} diff --git a/cmd/kk/cmd/completion/completion.go b/cmd/kk/cmd/completion/completion.go deleted file mode 100644 index ff832f418..000000000 --- a/cmd/kk/cmd/completion/completion.go +++ /dev/null @@ -1,113 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package completion - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" -) - -// CompletionOptions is the option of completion command -type CompletionOptions struct { - Type string -} - -func NewCompletionOptions() *CompletionOptions { - return &CompletionOptions{} -} - -// ShellTypes contains all types of shell -var ShellTypes = []string{ - "zsh", "bash", "powerShell", -} - -var completionOptions CompletionOptions - -func NewCmdCompletion() *cobra.Command { - o := NewCompletionOptions() - cmd := &cobra.Command{ - Use: "completion", - Short: "Generate shell completion scripts", - Long: `Generate shell completion scripts -Normally you don't need to do more extra work to have this feature if you've installed kk by brew`, - Example: `# Installing bash completion on Linux -## If bash-completion is not installed on Linux, please install the 'bash-completion' package -## via your distribution's package manager. -## Load the ks completion code for bash into the current shell -source <(ks completion bash) -## Write bash completion code to a file and source if from .bash_profile -mkdir -p ~/.config/kk/ && kk completion --type bash > ~/.config/kk/completion.bash.inc -printf " -# kk shell completion -source '$HOME/.config/kk/completion.bash.inc' -" >> $HOME/.bash_profile -source $HOME/.bash_profile - -In order to have good experience on zsh completion, ohmyzsh is a good choice. -Please install ohmyzsh by the following command -sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" -Get more details about onmyzsh from https://github.com/ohmyzsh/ohmyzsh - -Load the kk completion code for zsh[1] into the current shell -source <(kk completion --type zsh) -Set the kk completion code for zsh[1] to autoload on startup -kk completion --type zsh > "${fpath[1]}/_kk"`, - Run: func(cmd *cobra.Command, _ []string) { - util.CheckErr(o.Run(cmd)) - }, - } - - o.AddFlags(cmd) - if err := completionSetting(cmd); err != nil { - panic(fmt.Sprintf("register flag type for sub-command doc failed %#v\n", err)) - } - return cmd -} - -func (o *CompletionOptions) Run(cmd *cobra.Command) error { - var err error - shellType := completionOptions.Type - switch shellType { - case "zsh": - err = cmd.GenZshCompletion(cmd.OutOrStdout()) - case "powerShell": - err = cmd.GenPowerShellCompletion(cmd.OutOrStdout()) - case "bash": - err = cmd.GenBashCompletion(cmd.OutOrStdout()) - case "": - err = cmd.Help() - default: - err = fmt.Errorf("unknown shell type %s", shellType) - } - return err -} - -func (o *CompletionOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&completionOptions.Type, "type", "t", "", - fmt.Sprintf("Generate different types of shell which are %v", ShellTypes)) -} - -func completionSetting(cmd *cobra.Command) error { - err := cmd.RegisterFlagCompletionFunc("type", func(cmd *cobra.Command, args []string, toComplete string) ( - i []string, directive cobra.ShellCompDirective) { - return ShellTypes, cobra.ShellCompDirectiveDefault - }) - return err -} diff --git a/cmd/kk/cmd/create/cluster.go b/cmd/kk/cmd/create/cluster.go deleted file mode 100644 index 3ca16ec03..000000000 --- a/cmd/kk/cmd/create/cluster.go +++ /dev/null @@ -1,163 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package create - -import ( - "fmt" - "time" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/pipelines" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere" -) - -type CreateClusterOptions struct { - CommonOptions *options.CommonOptions - - ClusterCfgFile string - Kubernetes string - EnableKubeSphere bool - KubeSphere string - LocalStorage bool - SkipPullImages bool - SkipPushImages bool - SecurityEnhancement bool - ContainerManager string - DownloadCmd string - Artifact string - InstallPackages bool - - localStorageChanged bool -} - -func NewCreateClusterOptions() *CreateClusterOptions { - return &CreateClusterOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCreateCluster creates a new create cluster command -func NewCmdCreateCluster() *cobra.Command { - o := NewCreateClusterOptions() - cmd := &cobra.Command{ - Use: "cluster", - Short: "Create a Kubernetes or KubeSphere cluster", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(cmd, args)) - util.CheckErr(o.Validate(cmd, args)) - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - - if err := completionSetting(cmd); err != nil { - panic(fmt.Sprintf("Got error with the completion setting")) - } - return cmd -} - -func (o *CreateClusterOptions) Complete(cmd *cobra.Command, args []string) error { - var ksVersion string - if o.EnableKubeSphere && len(args) > 0 { - ksVersion = args[0] - } else { - ksVersion = kubesphere.Latest().Version - } - o.KubeSphere = ksVersion - - if o.Artifact == "" { - o.InstallPackages = false - o.SkipPushImages = false - } - - if cmd.Flags().Changed("with-local-storage") { - o.localStorageChanged = true - } - return nil -} - -func (o *CreateClusterOptions) Validate(_ *cobra.Command, _ []string) error { - switch o.ContainerManager { - case common.Docker, common.Containerd, common.Crio, common.Isula: - default: - return fmt.Errorf("unsupport container runtime [%s]", o.ContainerManager) - } - return nil -} - -func (o *CreateClusterOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KubernetesVersion: o.Kubernetes, - KsEnable: o.EnableKubeSphere, - KsVersion: o.KubeSphere, - SkipPullImages: o.SkipPullImages, - SkipPushImages: o.SkipPushImages, - SecurityEnhancement: o.SecurityEnhancement, - Debug: o.CommonOptions.Verbose, - IgnoreErr: o.CommonOptions.IgnoreErr, - SkipConfirmCheck: o.CommonOptions.SkipConfirmCheck, - ContainerManager: o.ContainerManager, - Artifact: o.Artifact, - InstallPackages: o.InstallPackages, - Namespace: o.CommonOptions.Namespace, - } - - if o.localStorageChanged { - deploy := o.LocalStorage - arg.DeployLocalStorage = &deploy - } - - return pipelines.CreateCluster(arg, o.DownloadCmd) -} - -func (o *CreateClusterOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") - cmd.Flags().BoolVarP(&o.LocalStorage, "with-local-storage", "", false, "Deploy a local PV provisioner") - cmd.Flags().BoolVarP(&o.EnableKubeSphere, "with-kubesphere", "", false, fmt.Sprintf("Deploy a specific version of kubesphere (default %s)", kubesphere.Latest().Version)) - cmd.Flags().BoolVarP(&o.SkipPullImages, "skip-pull-images", "", false, "Skip pre pull images") - cmd.Flags().BoolVarP(&o.SkipPushImages, "skip-push-images", "", false, "Skip pre push images") - cmd.Flags().BoolVarP(&o.SecurityEnhancement, "with-security-enhancement", "", false, "Security enhancement") - cmd.Flags().StringVarP(&o.ContainerManager, "container-manager", "", "docker", "Container runtime: docker, crio, containerd and isula.") - cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s", - `The user defined command to download the necessary binary files. The first param '%s' is output path, the second param '%s', is the URL`) - cmd.Flags().StringVarP(&o.Artifact, "artifact", "a", "", "Path to a KubeKey artifact") - cmd.Flags().BoolVarP(&o.InstallPackages, "with-packages", "", false, "install operation system packages by artifact") -} - -func completionSetting(cmd *cobra.Command) (err error) { - cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ( - strings []string, directive cobra.ShellCompDirective) { - versionArray := kubesphere.VersionsStringArr() - versionArray = append(versionArray, time.Now().Add(-time.Hour*24).Format("nightly-20060102")) - return versionArray, cobra.ShellCompDirectiveNoFileComp - } - - err = cmd.RegisterFlagCompletionFunc("with-kubernetes", func(cmd *cobra.Command, args []string, toComplete string) ( - strings []string, directive cobra.ShellCompDirective) { - return kubernetes.SupportedK8sVersionList(), cobra.ShellCompDirectiveNoFileComp - }) - return -} diff --git a/cmd/kk/cmd/create/config.go b/cmd/kk/cmd/create/config.go deleted file mode 100644 index ab259b24d..000000000 --- a/cmd/kk/cmd/create/config.go +++ /dev/null @@ -1,97 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package create - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/config" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere" -) - -type CreateConfigOptions struct { - CommonOptions *options.CommonOptions - Name string - ClusterCfgFile string - Kubernetes string - EnableKubeSphere bool - KubeSphere string - FromCluster bool - KubeConfig string -} - -func NewCreateConfigOptions() *CreateConfigOptions { - return &CreateConfigOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCreateConfig creates a create config command -func NewCmdCreateConfig() *cobra.Command { - o := NewCreateConfigOptions() - cmd := &cobra.Command{ - Use: "config", - Short: "Create cluster configuration file", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(cmd, args)) - util.CheckErr(o.Run()) - - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *CreateConfigOptions) Complete(cmd *cobra.Command, args []string) error { - var ksVersion string - if o.EnableKubeSphere && len(args) > 0 { - ksVersion = args[0] - } else { - ksVersion = kubesphere.Latest().Version - } - o.KubeSphere = ksVersion - return nil -} - -func (o *CreateConfigOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KubernetesVersion: o.Kubernetes, - KsEnable: o.EnableKubeSphere, - KsVersion: o.KubeSphere, - FromCluster: o.FromCluster, - KubeConfig: o.KubeConfig, - } - - return config.GenerateKubeKeyConfig(arg, o.Name) -} - -func (o *CreateConfigOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.Name, "name", "", "sample", "Specify a name of cluster object") - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Specify a configuration file path") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") - cmd.Flags().BoolVarP(&o.EnableKubeSphere, "with-kubesphere", "", false, fmt.Sprintf("Deploy a specific version of kubesphere (default %s)", kubesphere.Latest().Version)) - cmd.Flags().BoolVarP(&o.FromCluster, "from-cluster", "", false, "Create a configuration based on existing cluster") - cmd.Flags().StringVarP(&o.KubeConfig, "kubeconfig", "", "", "Specify a kubeconfig file") -} diff --git a/cmd/kk/cmd/create/create.go b/cmd/kk/cmd/create/create.go deleted file mode 100644 index dfb826e86..000000000 --- a/cmd/kk/cmd/create/create.go +++ /dev/null @@ -1,49 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package create - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" -) - -type CreateOptions struct { - CommonOptions *options.CommonOptions -} - -func NewCreateOptions() *CreateOptions { - return &CreateOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCreate creates a new create command -func NewCmdCreate() *cobra.Command { - o := NewCreateOptions() - cmd := &cobra.Command{ - Use: "create", - Short: "Create a cluster or a cluster configuration file", - } - - o.CommonOptions.AddCommonFlag(cmd) - - cmd.AddCommand(NewCmdCreateCluster()) - cmd.AddCommand(NewCmdCreateConfig()) - cmd.AddCommand(NewCmdCreateManifest()) - return cmd -} diff --git a/cmd/kk/cmd/create/manifest.go b/cmd/kk/cmd/create/manifest.go deleted file mode 100644 index 4ede87a3b..000000000 --- a/cmd/kk/cmd/create/manifest.go +++ /dev/null @@ -1,96 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package create - -import ( - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/pkg/errors" - "github.com/spf13/cobra" - "k8s.io/client-go/util/homedir" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/artifact" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" -) - -type CreateManifestOptions struct { - CommonOptions *options.CommonOptions - - Name string - KubeConfig string - FileName string -} - -func NewCreateManifestOptions() *CreateManifestOptions { - return &CreateManifestOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCreateManifest creates a create manifest command -func NewCmdCreateManifest() *cobra.Command { - o := NewCreateManifestOptions() - cmd := &cobra.Command{ - Use: "manifest", - Short: "Create an offline installation package configuration file", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(cmd, args)) - util.CheckErr(o.Run()) - - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *CreateManifestOptions) Complete(cmd *cobra.Command, args []string) error { - if o.Name != "" { - o.Name = strings.Split(o.Name, ".")[0] - } - if o.KubeConfig == "" { - o.KubeConfig = filepath.Join(homedir.HomeDir(), ".kube", "config") - } - if o.FileName == "" { - currentDir, err := filepath.Abs(filepath.Dir(os.Args[0])) - if err != nil { - return errors.Wrap(err, "Failed to get current dir") - } - o.FileName = filepath.Join(currentDir, fmt.Sprintf("manifest-%s.yaml", o.Name)) - } - return nil -} - -func (o *CreateManifestOptions) Run() error { - arg := common.Argument{ - FilePath: o.FileName, - KubeConfig: o.KubeConfig, - } - return artifact.CreateManifest(arg, o.Name) -} - -func (o *CreateManifestOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.Name, "name", "", "sample", "Specify a name of manifest object") - cmd.Flags().StringVarP(&o.FileName, "filename", "f", "", "Specify a manifest file path") - cmd.Flags().StringVar(&o.KubeConfig, "kubeconfig", "", "Specify a kubeconfig file") -} diff --git a/cmd/kk/cmd/create/phase/binary.go b/cmd/kk/cmd/create/phase/binary.go deleted file mode 100755 index f9d4b4aba..000000000 --- a/cmd/kk/cmd/create/phase/binary.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/binary" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes" -) - -type CreateBinaryOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - Kubernetes string - DownloadCmd string -} - -func NewCreateBinaryOptions() *CreateBinaryOptions { - return &CreateBinaryOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCreateBinary creates a new artifact import command -func NewCmdCreateBinary() *cobra.Command { - o := NewCreateBinaryOptions() - cmd := &cobra.Command{ - Use: "binary", - Short: "Download the binaries on the local", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - if err := k8sCompletionSetting(cmd); err != nil { - panic(fmt.Sprintf("Got error with the completion setting")) - } - return cmd -} - -func (o *CreateBinaryOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KubernetesVersion: o.Kubernetes, - Debug: o.CommonOptions.Verbose, - } - return binary.CreateBinary(arg, o.DownloadCmd) -} - -func (o *CreateBinaryOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") - cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s", - `The user defined command to download the necessary binary files. The first param '%s' is output path, the second param '%s', is the URL`) - -} - -func k8sCompletionSetting(cmd *cobra.Command) (err error) { - err = cmd.RegisterFlagCompletionFunc("with-kubernetes", func(cmd *cobra.Command, args []string, toComplete string) ( - strings []string, directive cobra.ShellCompDirective) { - return kubernetes.SupportedK8sVersionList(), cobra.ShellCompDirectiveNoFileComp - }) - return -} diff --git a/cmd/kk/cmd/create/phase/configure.go b/cmd/kk/cmd/create/phase/configure.go deleted file mode 100644 index 635f2ea4f..000000000 --- a/cmd/kk/cmd/create/phase/configure.go +++ /dev/null @@ -1,92 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/kubernetes" -) - -type CreateConfigureKubernetesOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - Kubernetes string - LocalStorage bool - - localStorageChanged bool -} - -func NewCreateConfigureKubernetesOptions() *CreateConfigureKubernetesOptions { - return &CreateConfigureKubernetesOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -func (o *CreateConfigureKubernetesOptions) Complete(cmd *cobra.Command, args []string) error { - if cmd.Flags().Changed("with-local-storage") { - o.localStorageChanged = true - } - return nil -} - -// NewCmdCreateConfigureKubernetes creates a new CreateConfigureKubernetes command -func NewCmdCreateConfigureKubernetes() *cobra.Command { - o := NewCreateConfigureKubernetesOptions() - cmd := &cobra.Command{ - Use: "configure", - Short: "Configure the k8s cluster with plugins, certs and PV", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(cmd, args)) - util.CheckErr(o.Run()) - }, - } - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - - if err := k8sCompletionSetting(cmd); err != nil { - panic(fmt.Sprintf("Got error with the completion setting")) - } - return cmd -} - -func (o *CreateConfigureKubernetesOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KubernetesVersion: o.Kubernetes, - Debug: o.CommonOptions.Verbose, - Namespace: o.CommonOptions.Namespace, - } - - if o.localStorageChanged { - deploy := o.LocalStorage - arg.DeployLocalStorage = &deploy - } - - return kubernetes.CreateConfigureKubernetes(arg) -} - -func (o *CreateConfigureKubernetesOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") - cmd.Flags().BoolVarP(&o.LocalStorage, "with-local-storage", "", false, "Deploy a local PV provisioner") -} diff --git a/cmd/kk/cmd/create/phase/etcd.go b/cmd/kk/cmd/create/phase/etcd.go deleted file mode 100755 index 7248164bf..000000000 --- a/cmd/kk/cmd/create/phase/etcd.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/etcd" -) - -type CreateEtcdOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string -} - -func NewCreateEtcdOptions() *CreateEtcdOptions { - return &CreateEtcdOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCreateEtcd creates a new install etcd command -func NewCmdCreateEtcd() *cobra.Command { - o := NewCreateEtcdOptions() - cmd := &cobra.Command{ - Use: "etcd", - Short: "Install the ETCD cluster on the master", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *CreateEtcdOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - Debug: o.CommonOptions.Verbose, - } - return etcd.CreateEtcd(arg) -} - -func (o *CreateEtcdOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") -} diff --git a/cmd/kk/cmd/create/phase/images.go b/cmd/kk/cmd/create/phase/images.go deleted file mode 100755 index 01c09685c..000000000 --- a/cmd/kk/cmd/create/phase/images.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/images" -) - -type CreateImagesOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - Kubernetes string - ContainerManager string -} - -func NewCreateImagesOptions() *CreateImagesOptions { - return &CreateImagesOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCreateImages creates a new Images command -func NewCmdCreateImages() *cobra.Command { - o := NewCreateImagesOptions() - cmd := &cobra.Command{ - Use: "images", - Short: "Down the container and pull the images before creating your cluster", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Validate(cmd, args)) - util.CheckErr(o.Run()) - }, - } - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - - if err := k8sCompletionSetting(cmd); err != nil { - panic(fmt.Sprintf("Got error with the completion setting")) - } - return cmd -} - -func (o *CreateImagesOptions) Validate(_ *cobra.Command, _ []string) error { - switch o.ContainerManager { - case common.Docker, common.Containerd, common.Crio, common.Isula: - default: - return fmt.Errorf("unsupport container runtime [%s]", o.ContainerManager) - } - return nil -} - -func (o *CreateImagesOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KubernetesVersion: o.Kubernetes, - ContainerManager: o.ContainerManager, - Debug: o.CommonOptions.Verbose, - } - return images.CreateImages(arg) -} - -func (o *CreateImagesOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") - cmd.Flags().StringVarP(&o.ContainerManager, "container-manager", "", "docker", "Container runtime: docker, crio, containerd and isula.") -} diff --git a/cmd/kk/cmd/create/phase/init.go b/cmd/kk/cmd/create/phase/init.go deleted file mode 100644 index 60f5c83b3..000000000 --- a/cmd/kk/cmd/create/phase/init.go +++ /dev/null @@ -1,75 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/kubernetes" -) - -type CreateInitClusterOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - Kubernetes string -} - -func NewCreateInitClusterOptions() *CreateInitClusterOptions { - return &CreateInitClusterOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdUpgrade creates a new upgrade command -func NewCmdCreateInitCluster() *cobra.Command { - o := NewCreateInitClusterOptions() - cmd := &cobra.Command{ - Use: "init", - Short: "Init the k8s cluster", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Run()) - }, - } - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - - if err := k8sCompletionSetting(cmd); err != nil { - panic(fmt.Sprintf("Got error with the completion setting")) - } - return cmd -} - -func (o *CreateInitClusterOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KubernetesVersion: o.Kubernetes, - Debug: o.CommonOptions.Verbose, - Namespace: o.CommonOptions.Namespace, - } - - return kubernetes.CreateInitCluster(arg) -} - -func (o *CreateInitClusterOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") -} diff --git a/cmd/kk/cmd/create/phase/join.go b/cmd/kk/cmd/create/phase/join.go deleted file mode 100644 index 31a442ac3..000000000 --- a/cmd/kk/cmd/create/phase/join.go +++ /dev/null @@ -1,75 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/kubernetes" -) - -type CreateJoinNodesOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - Kubernetes string -} - -func NewCreateJoinNodesOptions() *CreateJoinNodesOptions { - return &CreateJoinNodesOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCreateJoinNodes creates a new join nodes phase command -func NewCmdCreateJoinNodes() *cobra.Command { - o := NewCreateJoinNodesOptions() - cmd := &cobra.Command{ - Use: "join", - Short: "Join the control-plane nodes and worker nodes in the k8s cluster", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Run()) - }, - } - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - - if err := k8sCompletionSetting(cmd); err != nil { - panic(fmt.Sprintf("Got error with the completion setting")) - } - return cmd -} - -func (o *CreateJoinNodesOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KubernetesVersion: o.Kubernetes, - Debug: o.CommonOptions.Verbose, - Namespace: o.CommonOptions.Namespace, - } - - return kubernetes.CreateJoinNodes(arg) -} - -func (o *CreateJoinNodesOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") -} diff --git a/cmd/kk/cmd/create/phase/kubesphere.go b/cmd/kk/cmd/create/phase/kubesphere.go deleted file mode 100755 index 39f09d325..000000000 --- a/cmd/kk/cmd/create/phase/kubesphere.go +++ /dev/null @@ -1,101 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "fmt" - "time" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - alpha "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/kubesphere" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere" -) - -type CreateKubeSphereOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - EnableKubeSphere bool - KubeSphere string -} - -func NewCreateKubeSphereOptions() *CreateKubeSphereOptions { - return &CreateKubeSphereOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCreateKubeSphere creates a new CreateKubeSphere command -func NewCmdCreateKubeSphere() *cobra.Command { - o := NewCreateKubeSphereOptions() - cmd := &cobra.Command{ - Use: "kubesphere", - Short: "Install the kubesphere with the input version", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(cmd, args)) - util.CheckErr(o.Run()) - }, - } - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - - if err := ksCompletionSetting(cmd); err != nil { - panic(fmt.Sprintf("Got error with the completion setting")) - } - return cmd -} - -func (o *CreateKubeSphereOptions) Complete(cmd *cobra.Command, args []string) error { - var ksVersion string - if o.EnableKubeSphere && len(args) > 0 { - ksVersion = args[0] - } else { - ksVersion = kubesphere.Latest().Version - } - o.KubeSphere = ksVersion - return nil -} - -func (o *CreateKubeSphereOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KsEnable: o.EnableKubeSphere, - KsVersion: o.KubeSphere, - SkipConfirmCheck: o.CommonOptions.SkipConfirmCheck, - Debug: o.CommonOptions.Verbose, - } - return alpha.CreateKubeSphere(arg) -} - -func (o *CreateKubeSphereOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().BoolVarP(&o.EnableKubeSphere, "with-kubesphere", "", false, fmt.Sprintf("Deploy a specific version of kubesphere (default %s)", kubesphere.Latest().Version)) -} - -func ksCompletionSetting(cmd *cobra.Command) (err error) { - cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ( - strings []string, directive cobra.ShellCompDirective) { - versionArray := kubesphere.VersionsStringArr() - versionArray = append(versionArray, time.Now().Add(-time.Hour*24).Format("nightly-20060102")) - return versionArray, cobra.ShellCompDirectiveNoFileComp - } - - return -} diff --git a/cmd/kk/cmd/create/phase/os.go b/cmd/kk/cmd/create/phase/os.go deleted file mode 100755 index fc7eb76d8..000000000 --- a/cmd/kk/cmd/create/phase/os.go +++ /dev/null @@ -1,68 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/os" -) - -type ConfigOSOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - InstallPackages bool -} - -func NewConfigOSOptions() *ConfigOSOptions { - return &ConfigOSOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdConfigOS creates a new init os command -func NewCmdConfigOS() *cobra.Command { - o := NewConfigOSOptions() - cmd := &cobra.Command{ - Use: "os", - Short: "Init the os configure", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *ConfigOSOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - Debug: o.CommonOptions.Verbose, - InstallPackages: o.InstallPackages, - } - return os.ConfigOS(arg) -} - -func (o *ConfigOSOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().BoolVarP(&o.InstallPackages, "with-packages", "", false, "install operation system packages by artifact") -} diff --git a/cmd/kk/cmd/create/phase/phase.go b/cmd/kk/cmd/create/phase/phase.go deleted file mode 100755 index 655fddeae..000000000 --- a/cmd/kk/cmd/create/phase/phase.go +++ /dev/null @@ -1,39 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "github.com/spf13/cobra" -) - -func NewPhaseCommand() *cobra.Command { - cmds := &cobra.Command{ - Use: "phase", - Short: "KubeKey create phase", - Long: `This is the create phase run cmd`, - } - cmds.AddCommand(NewCmdCreateBinary()) - cmds.AddCommand(NewCmdConfigOS()) - cmds.AddCommand(NewCmdCreateImages()) - cmds.AddCommand(NewCmdCreateEtcd()) - cmds.AddCommand(NewCmdCreateInitCluster()) - cmds.AddCommand(NewCmdCreateJoinNodes()) - cmds.AddCommand(NewCmdCreateConfigureKubernetes()) - cmds.AddCommand(NewCmdCreateKubeSphere()) - - return cmds -} diff --git a/cmd/kk/cmd/delete/delete.go b/cmd/kk/cmd/delete/delete.go deleted file mode 100644 index 68777db41..000000000 --- a/cmd/kk/cmd/delete/delete.go +++ /dev/null @@ -1,48 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package delete - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" -) - -type DeleteOptions struct { - CommonOptions *options.CommonOptions -} - -func NewDeleteOptions() *DeleteOptions { - return &DeleteOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdDelete creates a new delete command -func NewCmdDelete() *cobra.Command { - o := NewDeleteOptions() - cmd := &cobra.Command{ - Use: "delete", - Short: "Delete node or cluster", - } - - o.CommonOptions.AddCommonFlag(cmd) - - cmd.AddCommand(NewCmdDeleteCluster()) - cmd.AddCommand(NewCmdDeleteNode()) - return cmd -} diff --git a/cmd/kk/cmd/delete/delete_cluster.go b/cmd/kk/cmd/delete/delete_cluster.go deleted file mode 100644 index 19fac4d23..000000000 --- a/cmd/kk/cmd/delete/delete_cluster.go +++ /dev/null @@ -1,72 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package delete - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/pipelines" -) - -type DeleteClusterOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - Kubernetes string - DeleteCRI bool -} - -func NewDeleteClusterOptions() *DeleteClusterOptions { - return &DeleteClusterOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdDeleteCluster creates a new delete cluster command -func NewCmdDeleteCluster() *cobra.Command { - o := NewDeleteClusterOptions() - cmd := &cobra.Command{ - Use: "cluster", - Short: "Delete a cluster", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *DeleteClusterOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - Debug: o.CommonOptions.Verbose, - KubernetesVersion: o.Kubernetes, - DeleteCRI: o.DeleteCRI, - SkipConfirmCheck: o.CommonOptions.SkipConfirmCheck, - } - return pipelines.DeleteCluster(arg) -} - -func (o *DeleteClusterOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") - cmd.Flags().BoolVarP(&o.DeleteCRI, "all", "A", false, "Delete total cri conficutation and data directories") -} diff --git a/cmd/kk/cmd/delete/delete_node.go b/cmd/kk/cmd/delete/delete_node.go deleted file mode 100644 index 115db96e8..000000000 --- a/cmd/kk/cmd/delete/delete_node.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package delete - -import ( - "strings" - - "github.com/pkg/errors" - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/pipelines" -) - -type DeleteNodeOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - nodeName string -} - -func NewDeleteNodeOptions() *DeleteNodeOptions { - return &DeleteNodeOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdDeleteNode creates a new delete node command -func NewCmdDeleteNode() *cobra.Command { - o := NewDeleteNodeOptions() - cmd := &cobra.Command{ - Use: "node", - Short: "delete a node", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(cmd, args)) - util.CheckErr(o.Validate()) - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *DeleteNodeOptions) Complete(cmd *cobra.Command, args []string) error { - o.nodeName = strings.Join(args, "") - return nil -} - -func (o *DeleteNodeOptions) Validate() error { - if o.nodeName == "" { - return errors.New("node name can not be empty") - } - return nil -} - -func (o *DeleteNodeOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - Debug: o.CommonOptions.Verbose, - NodeName: o.nodeName, - SkipConfirmCheck: o.CommonOptions.SkipConfirmCheck, - } - return pipelines.DeleteNode(arg) -} - -func (o *DeleteNodeOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - -} diff --git a/cmd/kk/cmd/init/init.go b/cmd/kk/cmd/init/init.go deleted file mode 100644 index cea0d54e3..000000000 --- a/cmd/kk/cmd/init/init.go +++ /dev/null @@ -1,46 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package init - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" -) - -type InitOptions struct { - CommonOptions *options.CommonOptions -} - -func NewInitOptions() *InitOptions { - return &InitOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdInit create a new init command -func NewCmdInit() *cobra.Command { - o := NewInitOptions() - cmd := &cobra.Command{ - Use: "init", - Short: "Initializes the installation environment", - } - o.CommonOptions.AddCommonFlag(cmd) - cmd.AddCommand(NewCmdInitOs()) - cmd.AddCommand(NewCmdInitRegistry()) - return cmd -} diff --git a/cmd/kk/cmd/init/init_os.go b/cmd/kk/cmd/init/init_os.go deleted file mode 100644 index 1b23bc8c8..000000000 --- a/cmd/kk/cmd/init/init_os.go +++ /dev/null @@ -1,68 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package init - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/pipelines" -) - -type InitOsOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - Artifact string -} - -func NewInitOsOptions() *InitOsOptions { - return &InitOsOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdInitOs creates a new init os command -func NewCmdInitOs() *cobra.Command { - o := NewInitOsOptions() - cmd := &cobra.Command{ - Use: "os", - Short: "Init operating system", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *InitOsOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - Debug: o.CommonOptions.Verbose, - Artifact: o.Artifact, - } - return pipelines.InitDependencies(arg) -} - -func (o *InitOsOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Artifact, "artifact", "a", "", "Path to a KubeKey artifact") -} diff --git a/cmd/kk/cmd/init/init_registry.go b/cmd/kk/cmd/init/init_registry.go deleted file mode 100644 index bfae0df96..000000000 --- a/cmd/kk/cmd/init/init_registry.go +++ /dev/null @@ -1,76 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package init - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/pipelines" -) - -type InitRegistryOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - DownloadCmd string - Artifact string -} - -func NewInitRegistryOptions() *InitRegistryOptions { - return &InitRegistryOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdInitRegistry creates a new init os command -func NewCmdInitRegistry() *cobra.Command { - o := NewInitRegistryOptions() - cmd := &cobra.Command{ - Use: "registry", - Short: "Init a local image registry", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(cmd, args)) - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *InitRegistryOptions) Complete(_ *cobra.Command, _ []string) error { - return nil -} - -func (o *InitRegistryOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - Debug: o.CommonOptions.Verbose, - Artifact: o.Artifact, - } - return pipelines.InitRegistry(arg, o.DownloadCmd) -} - -func (o *InitRegistryOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s", - `The user defined command to download the necessary files. The first param '%s' is output path, the second param '%s', is the URL`) - cmd.Flags().StringVarP(&o.Artifact, "artifact", "a", "", "Path to a KubeKey artifact") -} diff --git a/cmd/kk/cmd/options/common_options.go b/cmd/kk/cmd/options/common_options.go deleted file mode 100644 index 0badd11ba..000000000 --- a/cmd/kk/cmd/options/common_options.go +++ /dev/null @@ -1,39 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/cobra" -) - -type CommonOptions struct { - Verbose bool - SkipConfirmCheck bool - IgnoreErr bool - Namespace string -} - -func NewCommonOptions() *CommonOptions { - return &CommonOptions{} -} - -func (o *CommonOptions) AddCommonFlag(cmd *cobra.Command) { - cmd.Flags().BoolVar(&o.Verbose, "debug", false, "Print detailed information") - cmd.Flags().BoolVarP(&o.SkipConfirmCheck, "yes", "y", false, "Skip confirm check") - cmd.Flags().BoolVar(&o.IgnoreErr, "ignore-err", false, "Ignore the error message, remove the host which reported error and force to continue") - cmd.Flags().StringVar(&o.Namespace, "namespace", "kubekey-system", "KubeKey namespace to use") -} diff --git a/cmd/kk/cmd/options/io_options.go b/cmd/kk/cmd/options/io_options.go deleted file mode 100644 index 02532fd2b..000000000 --- a/cmd/kk/cmd/options/io_options.go +++ /dev/null @@ -1,30 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package options - -import "io" - -// IOStreams provides the standard names for iostreams. This is useful for embedding and for unit testing. -// Inconsistent and different names make it hard to read and review code -type IOStreams struct { - // In think, os.Stdin - In io.Reader - // Out think, os.Stdout - Out io.Writer - // ErrOut think, os.Stderr - ErrOut io.Writer -} diff --git a/cmd/kk/cmd/plugin/list.go b/cmd/kk/cmd/plugin/list.go deleted file mode 100644 index 0d7bd7b02..000000000 --- a/cmd/kk/cmd/plugin/list.go +++ /dev/null @@ -1,237 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package plugin - -import ( - "bytes" - "fmt" - "os" - "path/filepath" - "runtime" - "strings" - - "github.com/spf13/cobra" - cmdutil "k8s.io/kubectl/pkg/cmd/util" - "k8s.io/kubectl/pkg/util/i18n" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" -) - -type PluginListOptions struct { - Verifier PathVerifier - NameOnly bool - PluginPaths []string - options.IOStreams -} - -// NewCmdPluginList provides a way to list all plugin executables visible to kubectl -func NewCmdPluginList(streams options.IOStreams) *cobra.Command { - o := &PluginListOptions{ - IOStreams: streams, - } - - cmd := &cobra.Command{ - Use: "list", - Short: i18n.T("List all visible plugin executables on a user's PATH"), - Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(o.Complete(cmd)) - cmdutil.CheckErr(o.Run()) - }, - } - - cmd.Flags().BoolVar(&o.NameOnly, "name-only", o.NameOnly, "If true, display only the binary name of each plugin, rather than its full path") - return cmd -} - -func (o *PluginListOptions) Complete(cmd *cobra.Command) error { - o.Verifier = &CommandOverrideVerifier{ - root: cmd.Root(), - seenPlugins: make(map[string]string), - } - - o.PluginPaths = filepath.SplitList(os.Getenv("PATH")) - return nil -} - -func (o *PluginListOptions) Run() error { - pluginsFound := false - isFirstFile := true - pluginErrors := []error{} - pluginWarnings := 0 - - for _, dir := range uniquePathsList(o.PluginPaths) { - if len(strings.TrimSpace(dir)) == 0 { - continue - } - - files, err := os.ReadDir(dir) - if err != nil { - if _, ok := err.(*os.PathError); ok { - fmt.Fprintf(o.ErrOut, "Unable to read directory %q from your PATH: %v. Skipping...\n", dir, err) - continue - } - - pluginErrors = append(pluginErrors, fmt.Errorf("error: unable to read directory %q in your PATH: %v", dir, err)) - continue - } - - for _, f := range files { - if f.IsDir() { - continue - } - if !hasValidPrefix(f.Name(), ValidPluginFilenamePrefixes) { - continue - } - - if isFirstFile { - fmt.Fprintf(o.Out, "The following compatible plugins are available:\n\n") - pluginsFound = true - isFirstFile = false - } - - pluginPath := f.Name() - if !o.NameOnly { - pluginPath = filepath.Join(dir, pluginPath) - } - - fmt.Fprintf(o.Out, "%s\n", pluginPath) - if errs := o.Verifier.Verify(filepath.Join(dir, f.Name())); len(errs) != 0 { - for _, err := range errs { - fmt.Fprintf(o.ErrOut, " - %s\n", err) - pluginWarnings++ - } - } - } - } - - if !pluginsFound { - pluginErrors = append(pluginErrors, fmt.Errorf("error: unable to find any kubectl plugins in your PATH")) - } - - if pluginWarnings > 0 { - if pluginWarnings == 1 { - pluginErrors = append(pluginErrors, fmt.Errorf("error: one plugin warning was found")) - } else { - pluginErrors = append(pluginErrors, fmt.Errorf("error: %v plugin warnings were found", pluginWarnings)) - } - } - if len(pluginErrors) > 0 { - errs := bytes.NewBuffer(nil) - for _, e := range pluginErrors { - fmt.Fprintln(errs, e) - } - return fmt.Errorf("%s", errs.String()) - } - - return nil -} - -type PathVerifier interface { - // Verify determines if a given path is valid - Verify(path string) []error -} - -type CommandOverrideVerifier struct { - root *cobra.Command - seenPlugins map[string]string -} - -// Verify implements PathVerifier and determines if a given path -// is valid depending on whether or not it overwrites an existing -// kubectl command path, or a previously seen plugin. -func (v *CommandOverrideVerifier) Verify(path string) []error { - if v.root == nil { - return []error{fmt.Errorf("unable to verify path with nil root")} - } - - // extract the plugin binary name - segs := strings.Split(path, "/") - binName := segs[len(segs)-1] - - cmdPath := strings.Split(binName, "-") - if len(cmdPath) > 1 { - // the first argument is always "kubectl" for a plugin binary - cmdPath = cmdPath[1:] - } - - errors := []error{} - - if isExec, err := isExecutable(path); err == nil && !isExec { - errors = append(errors, fmt.Errorf("warning: %s identified as a kubectl plugin, but it is not executable", path)) - } else if err != nil { - errors = append(errors, fmt.Errorf("error: unable to identify %s as an executable file: %v", path, err)) - } - - if existingPath, ok := v.seenPlugins[binName]; ok { - errors = append(errors, fmt.Errorf("warning: %s is overshadowed by a similarly named plugin: %s", path, existingPath)) - } else { - v.seenPlugins[binName] = path - } - - if cmd, _, err := v.root.Find(cmdPath); err == nil { - errors = append(errors, fmt.Errorf("warning: %s overwrites existing command: %q", binName, cmd.CommandPath())) - } - - return errors -} - -func isExecutable(fullPath string) (bool, error) { - info, err := os.Stat(fullPath) - if err != nil { - return false, err - } - - if runtime.GOOS == "windows" { - fileExt := strings.ToLower(filepath.Ext(fullPath)) - - switch fileExt { - case ".bat", ".cmd", ".com", ".exe", ".ps1": - return true, nil - } - return false, nil - } - - if m := info.Mode(); !m.IsDir() && m&0111 != 0 { - return true, nil - } - - return false, nil -} - -// uniquePathsList deduplicates a given slice of strings without -// sorting or otherwise altering its order in any way. -func uniquePathsList(paths []string) []string { - seen := map[string]bool{} - newPaths := []string{} - for _, p := range paths { - if seen[p] { - continue - } - seen[p] = true - newPaths = append(newPaths, p) - } - return newPaths -} - -func hasValidPrefix(filepath string, validPrefixes []string) bool { - for _, prefix := range validPrefixes { - if strings.HasPrefix(filepath, prefix+"-") { - return true - } - } - return false -} diff --git a/cmd/kk/cmd/plugin/plugin.go b/cmd/kk/cmd/plugin/plugin.go deleted file mode 100644 index cabf9dde3..000000000 --- a/cmd/kk/cmd/plugin/plugin.go +++ /dev/null @@ -1,38 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package plugin - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" -) - -var ( - ValidPluginFilenamePrefixes = []string{"kk"} -) - -func NewCmdPlugin(streams options.IOStreams) *cobra.Command { - cmd := &cobra.Command{ - Use: "plugin [flags]", - DisableFlagsInUseLine: true, - Short: "Provides utilities for interacting with plugins", - } - - cmd.AddCommand(NewCmdPluginList(streams)) - return cmd -} diff --git a/cmd/kk/cmd/root.go b/cmd/kk/cmd/root.go deleted file mode 100755 index 3ca697a9b..000000000 --- a/cmd/kk/cmd/root.go +++ /dev/null @@ -1,230 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "fmt" - "os" - "os/exec" - "runtime" - "strings" - "syscall" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/add" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/alpha" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/artifact" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/cert" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/completion" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/create" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/delete" - initOs "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/init" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/plugin" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/upgrade" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/version" -) - -type KubeKeyOptions struct { - PluginHandler PluginHandler - Arguments []string - - options.IOStreams -} - -func NewDefaultKubeKeyCommand() *cobra.Command { - return NewDefaultKubeKeyCommandWithArgs(KubeKeyOptions{ - PluginHandler: NewDefaultPluginHandler(plugin.ValidPluginFilenamePrefixes), - Arguments: os.Args, - - IOStreams: options.IOStreams{In: os.Stdin, Out: os.Stdout, ErrOut: os.Stderr}, - }) -} - -func NewDefaultKubeKeyCommandWithArgs(o KubeKeyOptions) *cobra.Command { - cmd := NewKubeKeyCommand(o) - - if o.PluginHandler == nil { - return cmd - } - - if len(o.Arguments) > 1 { - cmdPathPieces := o.Arguments[1:] - - // only look for suitable extension executables if - // the specified command does not already exist - if _, _, err := cmd.Find(cmdPathPieces); err != nil { - // Also check the commands that will be added by Cobra. - // These commands are only added once rootCmd.Execute() is called, so we - // need to check them explicitly here. - var cmdName string // first "non-flag" arguments - for _, arg := range cmdPathPieces { - if !strings.HasPrefix(arg, "-") { - cmdName = arg - break - } - } - - switch cmdName { - case "help", cobra.ShellCompRequestCmd, cobra.ShellCompNoDescRequestCmd: - // Don't search for a plugin - default: - if err := HandlePluginCommand(o.PluginHandler, cmdPathPieces); err != nil { - os.Exit(1) - } - } - } - } - - return cmd -} - -// NewKubeKeyCommand creates a new kubekey root command -func NewKubeKeyCommand(o KubeKeyOptions) *cobra.Command { - cmds := &cobra.Command{ - Use: "kk", - Short: "Kubernetes/KubeSphere Deploy Tool", - Long: `Deploy a Kubernetes or KubeSphere cluster efficiently, flexibly and easily. There are three scenarios to use KubeKey. -1. Install Kubernetes only -2. Install Kubernetes and KubeSphere together in one command -3. Install Kubernetes first, then deploy KubeSphere on it using https://github.com/kubesphere/ks-installer`, - } - - cmds.AddCommand(initOs.NewCmdInit()) - - cmds.AddCommand(alpha.NewAlphaCmd()) - - cmds.AddCommand(create.NewCmdCreate()) - cmds.AddCommand(delete.NewCmdDelete()) - cmds.AddCommand(add.NewCmdAdd()) - cmds.AddCommand(upgrade.NewCmdUpgrade()) - cmds.AddCommand(cert.NewCmdCerts()) - cmds.AddCommand(artifact.NewCmdArtifact()) - - cmds.AddCommand(plugin.NewCmdPlugin(o.IOStreams)) - - cmds.AddCommand(completion.NewCmdCompletion()) - cmds.AddCommand(version.NewCmdVersion()) - - return cmds -} - -// PluginHandler is capable of parsing command line arguments -// and performing executable filename lookups to search -// for valid plugin files, and execute found plugins. -type PluginHandler interface { - // exists at the given filename, or a boolean false. - // Lookup will iterate over a list of given prefixes - // in order to recognize valid plugin filenames. - // The first filepath to match a prefix is returned. - Lookup(filename string) (string, bool) - // Execute receives an executable's filepath, a slice - // of arguments, and a slice of environment variables - // to relay to the executable. - Execute(executablePath string, cmdArgs, environment []string) error -} - -// DefaultPluginHandler implements PluginHandler -type DefaultPluginHandler struct { - ValidPrefixes []string -} - -// NewDefaultPluginHandler instantiates the DefaultPluginHandler with a list of -// given filename prefixes used to identify valid plugin filenames. -func NewDefaultPluginHandler(validPrefixes []string) *DefaultPluginHandler { - return &DefaultPluginHandler{ - ValidPrefixes: validPrefixes, - } -} - -// Lookup implements PluginHandler -func (h *DefaultPluginHandler) Lookup(filename string) (string, bool) { - for _, prefix := range h.ValidPrefixes { - path, err := exec.LookPath(fmt.Sprintf("%s-%s", prefix, filename)) - if err != nil || len(path) == 0 { - continue - } - return path, true - } - - return "", false -} - -// Execute implements PluginHandler -func (h *DefaultPluginHandler) Execute(executablePath string, cmdArgs, environment []string) error { - - // Windows does not support exec syscall. - if runtime.GOOS == "windows" { - cmd := exec.Command(executablePath, cmdArgs...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Stdin = os.Stdin - cmd.Env = environment - err := cmd.Run() - if err == nil { - os.Exit(0) - } - return err - } - - // invoke cmd binary relaying the environment and args given - // append executablePath to cmdArgs, as execve will make first argument the "binary name". - return syscall.Exec(executablePath, append([]string{executablePath}, cmdArgs...), environment) -} - -// HandlePluginCommand receives a pluginHandler and command-line arguments and attempts to find -// a plugin executable on the PATH that satisfies the given arguments. -func HandlePluginCommand(pluginHandler PluginHandler, cmdArgs []string) error { - var remainingArgs []string // all "non-flag" arguments - for _, arg := range cmdArgs { - if strings.HasPrefix(arg, "-") { - break - } - remainingArgs = append(remainingArgs, strings.Replace(arg, "-", "_", -1)) - } - - if len(remainingArgs) == 0 { - // the length of cmdArgs is at least 1 - return fmt.Errorf("flags cannot be placed before plugin name: %s", cmdArgs[0]) - } - - foundBinaryPath := "" - - // attempt to find binary, starting at longest possible name with given cmdArgs - for len(remainingArgs) > 0 { - path, found := pluginHandler.Lookup(strings.Join(remainingArgs, "-")) - if !found { - remainingArgs = remainingArgs[:len(remainingArgs)-1] - continue - } - - foundBinaryPath = path - break - } - - if len(foundBinaryPath) == 0 { - return nil - } - - // invoke cmd binary relaying the current environment and args given - if err := pluginHandler.Execute(foundBinaryPath, cmdArgs[len(remainingArgs):], os.Environ()); err != nil { - return err - } - - return nil -} diff --git a/cmd/kk/cmd/upgrade/phase/binary.go b/cmd/kk/cmd/upgrade/phase/binary.go deleted file mode 100755 index 2e2bdc628..000000000 --- a/cmd/kk/cmd/upgrade/phase/binary.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/binary" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes" -) - -type UpgradeBinaryOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - Kubernetes string - DownloadCmd string -} - -func NewUpgradeBinaryOptions() *UpgradeBinaryOptions { - return &UpgradeBinaryOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdUpgradeBinary creates a new artifact import command -func NewCmdUpgradeBinary() *cobra.Command { - o := NewUpgradeBinaryOptions() - cmd := &cobra.Command{ - Use: "binary", - Short: "Download the binary and synchronize kubernetes binaries", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - if err := k8sCompletionSetting(cmd); err != nil { - panic(fmt.Sprintf("Got error with the completion setting")) - } - return cmd -} - -func (o *UpgradeBinaryOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KubernetesVersion: o.Kubernetes, - Debug: o.CommonOptions.Verbose, - } - return binary.UpgradeBinary(arg, o.DownloadCmd) -} - -func (o *UpgradeBinaryOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") - cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s", - `The user defined command to download the necessary binary files. The first param '%s' is output path, the second param '%s', is the URL`) - -} - -func k8sCompletionSetting(cmd *cobra.Command) (err error) { - err = cmd.RegisterFlagCompletionFunc("with-kubernetes", func(cmd *cobra.Command, args []string, toComplete string) ( - strings []string, directive cobra.ShellCompDirective) { - return kubernetes.SupportedK8sVersionList(), cobra.ShellCompDirectiveNoFileComp - }) - return -} diff --git a/cmd/kk/cmd/upgrade/phase/images.go b/cmd/kk/cmd/upgrade/phase/images.go deleted file mode 100755 index 86bd97a44..000000000 --- a/cmd/kk/cmd/upgrade/phase/images.go +++ /dev/null @@ -1,73 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/images" -) - -type UpgradeImagesOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - Kubernetes string -} - -func NewUpgradeImagesOptions() *UpgradeImagesOptions { - return &UpgradeImagesOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdUpgrade creates a new upgrade command -func NewCmdUpgradeImages() *cobra.Command { - o := NewUpgradeImagesOptions() - cmd := &cobra.Command{ - Use: "images", - Short: "Pull the images before upgrading your cluster", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Run()) - }, - } - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - - if err := k8sCompletionSetting(cmd); err != nil { - panic(fmt.Sprintf("Got error with the completion setting")) - } - return cmd -} - -func (o *UpgradeImagesOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KubernetesVersion: o.Kubernetes, - Debug: o.CommonOptions.Verbose, - } - return images.UpgradeImages(arg) -} - -func (o *UpgradeImagesOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") -} diff --git a/cmd/kk/cmd/upgrade/phase/kubesphere.go b/cmd/kk/cmd/upgrade/phase/kubesphere.go deleted file mode 100755 index 003b2307e..000000000 --- a/cmd/kk/cmd/upgrade/phase/kubesphere.go +++ /dev/null @@ -1,100 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "fmt" - "time" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - alpha "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/kubesphere" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere" -) - -type UpgradeKubeSphereOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - EnableKubeSphere bool - KubeSphere string -} - -func NewUpgradeKubeSphereOptions() *UpgradeKubeSphereOptions { - return &UpgradeKubeSphereOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdUpgradeKubeSphere creates a new UpgradeKubeSphere command -func NewCmdUpgradeKubeSphere() *cobra.Command { - o := NewUpgradeKubeSphereOptions() - cmd := &cobra.Command{ - Use: "kubesphere", - Short: "Upgrade your kubesphere to a newer version with this command", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(cmd, args)) - util.CheckErr(o.Run()) - }, - } - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - - if err := ksCompletionSetting(cmd); err != nil { - panic(fmt.Sprintf("Got error with the completion setting")) - } - return cmd -} - -func (o *UpgradeKubeSphereOptions) Complete(cmd *cobra.Command, args []string) error { - var ksVersion string - if o.EnableKubeSphere && len(args) > 0 { - ksVersion = args[0] - } else { - ksVersion = kubesphere.Latest().Version - } - o.KubeSphere = ksVersion - return nil -} - -func (o *UpgradeKubeSphereOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KsEnable: o.EnableKubeSphere, - KsVersion: o.KubeSphere, - SkipConfirmCheck: o.CommonOptions.SkipConfirmCheck, - Debug: o.CommonOptions.Verbose, - } - return alpha.UpgradeKubeSphere(arg) -} - -func (o *UpgradeKubeSphereOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().BoolVarP(&o.EnableKubeSphere, "with-kubesphere", "", false, fmt.Sprintf("Deploy a specific version of kubesphere (default %s)", kubesphere.Latest().Version)) -} - -func ksCompletionSetting(cmd *cobra.Command) (err error) { - cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ( - strings []string, directive cobra.ShellCompDirective) { - versionArray := kubesphere.VersionsStringArr() - versionArray = append(versionArray, time.Now().Add(-time.Hour*24).Format("nightly-20060102")) - return versionArray, cobra.ShellCompDirectiveNoFileComp - } - return -} diff --git a/cmd/kk/cmd/upgrade/phase/nodes.go b/cmd/kk/cmd/upgrade/phase/nodes.go deleted file mode 100755 index 83b0eaa29..000000000 --- a/cmd/kk/cmd/upgrade/phase/nodes.go +++ /dev/null @@ -1,73 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/nodes" -) - -type UpgradeNodesOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - Kubernetes string -} - -func NewUpgradeNodesOptions() *UpgradeNodesOptions { - return &UpgradeNodesOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdUpgrade creates a new upgrade command -func NewCmdUpgradeNodes() *cobra.Command { - o := NewUpgradeNodesOptions() - cmd := &cobra.Command{ - Use: "nodes", - Short: "Upgrade cluster on master nodes and worker nodes to the version you input", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Run()) - }, - } - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - - if err := k8sCompletionSetting(cmd); err != nil { - panic(fmt.Sprintf("Got error with the completion setting")) - } - return cmd -} - -func (o *UpgradeNodesOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KubernetesVersion: o.Kubernetes, - Debug: o.CommonOptions.Verbose, - } - return nodes.UpgradeNodes(arg) -} - -func (o *UpgradeNodesOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") -} diff --git a/cmd/kk/cmd/upgrade/phase/phase.go b/cmd/kk/cmd/upgrade/phase/phase.go deleted file mode 100755 index f17d1c7be..000000000 --- a/cmd/kk/cmd/upgrade/phase/phase.go +++ /dev/null @@ -1,36 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "github.com/spf13/cobra" -) - -func NewPhaseCommand() *cobra.Command { - cmds := &cobra.Command{ - Use: "phase", - Short: "KubeKey upgrade phase", - Long: `This is the upgrade phase run cmd`, - } - - cmds.AddCommand(NewCmdUpgradeBinary()) - cmds.AddCommand(NewCmdUpgradeImages()) - cmds.AddCommand(NewCmdUpgradeNodes()) - cmds.AddCommand(NewCmdUpgradeKubeSphere()) - - return cmds -} diff --git a/cmd/kk/cmd/upgrade/upgrade.go b/cmd/kk/cmd/upgrade/upgrade.go deleted file mode 100644 index 4e9f60381..000000000 --- a/cmd/kk/cmd/upgrade/upgrade.go +++ /dev/null @@ -1,118 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "fmt" - "time" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/pipelines" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere" -) - -type UpgradeOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - Kubernetes string - EnableKubeSphere bool - KubeSphere string - SkipPullImages bool - DownloadCmd string - Artifact string -} - -func NewUpgradeOptions() *UpgradeOptions { - return &UpgradeOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdUpgrade creates a new upgrade command -func NewCmdUpgrade() *cobra.Command { - o := NewUpgradeOptions() - cmd := &cobra.Command{ - Use: "upgrade", - Short: "Upgrade your cluster smoothly to a newer version with this command", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(cmd, args)) - util.CheckErr(o.Run()) - }, - } - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - - if err := completionSetting(cmd); err != nil { - panic(fmt.Sprintf("Got error with the completion setting")) - } - return cmd -} - -func (o *UpgradeOptions) Complete(cmd *cobra.Command, args []string) error { - var ksVersion string - if o.EnableKubeSphere && len(args) > 0 { - ksVersion = args[0] - } else { - ksVersion = kubesphere.Latest().Version - } - o.KubeSphere = ksVersion - return nil -} - -func (o *UpgradeOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KubernetesVersion: o.Kubernetes, - KsEnable: o.EnableKubeSphere, - KsVersion: o.KubeSphere, - SkipPullImages: o.SkipPullImages, - Debug: o.CommonOptions.Verbose, - SkipConfirmCheck: o.CommonOptions.SkipConfirmCheck, - Artifact: o.Artifact, - } - return pipelines.UpgradeCluster(arg, o.DownloadCmd) -} - -func (o *UpgradeOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") - cmd.Flags().BoolVarP(&o.EnableKubeSphere, "with-kubesphere", "", false, fmt.Sprintf("Deploy a specific version of kubesphere (default %s)", kubesphere.Latest().Version)) - cmd.Flags().BoolVarP(&o.SkipPullImages, "skip-pull-images", "", false, "Skip pre pull images") - cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s", - `The user defined command to download the necessary binary files. The first param '%s' is output path, the second param '%s', is the URL`) - cmd.Flags().StringVarP(&o.Artifact, "artifact", "a", "", "Path to a KubeKey artifact") -} - -func completionSetting(cmd *cobra.Command) (err error) { - cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ( - strings []string, directive cobra.ShellCompDirective) { - versionArray := kubesphere.VersionsStringArr() - versionArray = append(versionArray, time.Now().Add(-time.Hour*24).Format("nightly-20060102")) - return versionArray, cobra.ShellCompDirectiveNoFileComp - } - - err = cmd.RegisterFlagCompletionFunc("with-kubernetes", func(cmd *cobra.Command, args []string, toComplete string) ( - strings []string, directive cobra.ShellCompDirective) { - return kubernetes.SupportedK8sVersionList(), cobra.ShellCompDirectiveNoFileComp - }) - return -} diff --git a/cmd/kk/cmd/util/helpers.go b/cmd/kk/cmd/util/helpers.go deleted file mode 100644 index 1f4bffc2f..000000000 --- a/cmd/kk/cmd/util/helpers.go +++ /dev/null @@ -1,61 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package util - -import ( - "fmt" - "os" - "strings" -) - -const DefaultErrorExitCode = 1 - -var ErrExit = fmt.Errorf("exit") - -func fatal(msg string, code int) { - if len(msg) > 0 { - // add newline if needed - if !strings.HasSuffix(msg, "\n") { - msg += "\n" - } - fmt.Fprint(os.Stderr, msg) - } - os.Exit(code) -} - -var fatalErrHandler = fatal - -func CheckErr(err error) { - checkErr(err, fatalErrHandler) -} - -func checkErr(err error, handleErr func(string, int)) { - if err == nil { - return - } - - switch { - case err == ErrExit: - handleErr("", DefaultErrorExitCode) - default: - msg := err.Error() - if !strings.HasPrefix(msg, "error: ") { - msg = fmt.Sprintf("error: %s", msg) - } - handleErr(msg, DefaultErrorExitCode) - } -} diff --git a/cmd/kk/cmd/version/version.go b/cmd/kk/cmd/version/version.go deleted file mode 100644 index 13b990701..000000000 --- a/cmd/kk/cmd/version/version.go +++ /dev/null @@ -1,102 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package version - -import ( - "encoding/json" - "fmt" - "io" - "strings" - - "github.com/pkg/errors" - "github.com/spf13/cobra" - "gopkg.in/yaml.v3" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes" - "github.com/kubesphere/kubekey/v3/version" -) - -type Version struct { - KK *version.Info `json:"kk"` -} - -type VersionOptions struct { - Output string - ShowSupportedK8sVersionList bool -} - -func NewVersionOptions() *VersionOptions { - return &VersionOptions{} -} - -// NewCmdVersion creates a new version command -func NewCmdVersion() *cobra.Command { - o := NewVersionOptions() - - cmd := &cobra.Command{ - Use: "version", - Short: "print the client version information", - RunE: func(cmd *cobra.Command, _ []string) error { - if o.ShowSupportedK8sVersionList { - return printSupportedK8sVersionList(cmd.OutOrStdout()) - } - return o.Run() - }, - } - o.AddFlags(cmd) - return cmd -} - -func (o *VersionOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.Output, "output", "o", "", "Output format; available options are 'yaml', 'json' and 'short'") - cmd.Flags().BoolVarP(&o.ShowSupportedK8sVersionList, "show-supported-k8s", "", false, - `print the version of supported k8s`) -} - -func (o *VersionOptions) Run() error { - clientVersion := version.Get() - v := Version{ - KK: &clientVersion, - } - - switch o.Output { - case "": - fmt.Printf("kk version: %#v\n", v.KK) - case "short": - fmt.Printf("%s\n", v.KK.GitVersion) - case "yaml": - y, err := yaml.Marshal(&v) - if err != nil { - return err - } - fmt.Print(string(y)) - case "json": - y, err := json.MarshalIndent(&v, "", " ") - if err != nil { - return err - } - fmt.Println(string(y)) - default: - return errors.Errorf("invalid output format: %s", o.Output) - } - return nil -} - -func printSupportedK8sVersionList(output io.Writer) (err error) { - _, err = output.Write([]byte(fmt.Sprintln(strings.Join(kubernetes.SupportedK8sVersionList(), "\n")))) - return -} diff --git a/cmd/kk/kubekey.go b/cmd/kk/kubekey.go new file mode 100644 index 000000000..b11a7910b --- /dev/null +++ b/cmd/kk/kubekey.go @@ -0,0 +1,37 @@ +/* +Copyright 2023 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/kubesphere/kubekey/v4/cmd/kk/app" +) + +func main() { + if err := app.NewRootCommand().Execute(); err != nil { + vFlag := flag.Lookup("v") + if vFlag != nil { + fmt.Printf("%+v", err) + } else { + fmt.Printf("%v", err) + } + os.Exit(1) + } +} diff --git a/cmd/kk/main.go b/cmd/kk/main.go deleted file mode 100644 index 98f575e45..000000000 --- a/cmd/kk/main.go +++ /dev/null @@ -1,38 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "os" - "os/exec" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd" -) - -// Using a separate entry-point can reduce the size of the binary file -func main() { - c := cmd.NewDefaultKubeKeyCommand() - _ = exec.Command("/bin/bash", "-c", "ulimit -u 65535").Run() - _ = exec.Command("/bin/bash", "-c", "ulimit -n 65535").Run() - - // Execute adds all child commands to the root command and sets flags appropriately. - // This is called by main.main(). It only needs to happen once to the rootCmd. - if err := c.Execute(); err != nil { - //fmt.Println(err) - os.Exit(1) - } -} diff --git a/cmd/kk/pkg/addons/addons.go b/cmd/kk/pkg/addons/addons.go deleted file mode 100644 index 6d145e561..000000000 --- a/cmd/kk/pkg/addons/addons.go +++ /dev/null @@ -1,67 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package addons - -import ( - "net/url" - "os" - "path/filepath" - "strings" - - "github.com/pkg/errors" - "helm.sh/helm/v3/pkg/cli" - "helm.sh/helm/v3/pkg/getter" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" -) - -func InstallAddons(kubeConf *common.KubeConf, addon *kubekeyapiv1alpha2.Addon, kubeConfig string) error { - // install chart - if addon.Sources.Chart.Name != "" { - _ = os.Setenv("HELM_NAMESPACE", strings.TrimSpace(addon.Namespace)) - if err := InstallChart(kubeConf, addon, kubeConfig); err != nil { - return err - } - } - - // install yaml - if len(addon.Sources.Yaml.Path) != 0 { - var settings = cli.New() - p := getter.All(settings) - for _, yaml := range addon.Sources.Yaml.Path { - u, _ := url.Parse(yaml) - _, err := p.ByScheme(u.Scheme) - if err != nil { - fp, err := filepath.Abs(yaml) - if err != nil { - return errors.Wrap(err, "Failed to look up current directory") - } - yamlPaths := []string{fp} - if err := InstallYaml(yamlPaths, addon.Namespace, kubeConfig, kubeConf.Cluster.Kubernetes.Version); err != nil { - return err - } - } else { - yamlPaths := []string{yaml} - if err := InstallYaml(yamlPaths, addon.Namespace, kubeConfig, kubeConf.Cluster.Kubernetes.Version); err != nil { - return err - } - } - } - } - return nil -} diff --git a/cmd/kk/pkg/addons/charts.go b/cmd/kk/pkg/addons/charts.go deleted file mode 100644 index 176d3acff..000000000 --- a/cmd/kk/pkg/addons/charts.go +++ /dev/null @@ -1,246 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package addons - -import ( - "fmt" - "log" - "os" - "path/filepath" - "time" - - "github.com/pkg/errors" - "helm.sh/helm/v3/pkg/action" - "helm.sh/helm/v3/pkg/chart" - helmLoader "helm.sh/helm/v3/pkg/chart/loader" - "helm.sh/helm/v3/pkg/cli" - "helm.sh/helm/v3/pkg/cli/values" - "helm.sh/helm/v3/pkg/downloader" - "helm.sh/helm/v3/pkg/getter" - "helm.sh/helm/v3/pkg/release" - "helm.sh/helm/v3/pkg/storage/driver" - "k8s.io/client-go/util/homedir" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" -) - -func debug(format string, v ...interface{}) { - if false { - format = fmt.Sprintf("[debug] %s\n", format) - _ = log.Output(2, fmt.Sprintf(format, v...)) - } -} - -func InstallChart(kubeConf *common.KubeConf, addon *kubekeyapiv1alpha2.Addon, kubeConfig string) error { - actionConfig := new(action.Configuration) - var settings = cli.New() - helmDriver := os.Getenv("HELM_DRIVER") - settings.KubeConfig = kubeConfig - var namespace string - if addon.Namespace != "" { - namespace = addon.Namespace - } else { - namespace = "default" - } - - if err := actionConfig.Init(settings.RESTClientGetter(), namespace, helmDriver, debug); err != nil { - logger.Log.Fatal(err) - } - - valueOpts := &values.Options{} - if len(addon.Sources.Chart.Values) != 0 { - valueOpts.Values = addon.Sources.Chart.Values - } - if len(addon.Sources.Chart.ValuesFile) != 0 { - valueOpts.ValueFiles = []string{addon.Sources.Chart.ValuesFile} - } - - client := action.NewUpgrade(actionConfig) - - var chartName string - if addon.Sources.Chart.Name != "" { - if addon.Sources.Chart.Repo == "" && addon.Sources.Chart.Path != "" { - fmt.Println(addon.Sources.Chart.Repo) - chartName = filepath.Join(addon.Sources.Chart.Path, addon.Sources.Chart.Name) - } else { - chartName = addon.Sources.Chart.Name - } - } else { - logger.Log.Fatalln("No chart name is specified") - } - - args := []string{addon.Name, chartName} - - client.Install = true - client.Namespace = namespace - client.Timeout = 300 * time.Second - client.Keyring = defaultKeyring() - client.RepoURL = addon.Sources.Chart.Repo - client.Version = addon.Sources.Chart.Version - client.Wait = addon.Sources.Chart.Wait - //client.Force = true - - if client.Version == "" && client.Devel { - client.Version = ">0.0.0-0" - } - - if client.Install { - histClient := action.NewHistory(actionConfig) - histClient.Max = 1 - if _, err := histClient.Run(addon.Name); err == driver.ErrReleaseNotFound { - fmt.Printf("Release %q does not exist. Installing it now.\n", addon.Name) - instClient := action.NewInstall(actionConfig) - instClient.CreateNamespace = true - instClient.Namespace = client.Namespace - instClient.Timeout = client.Timeout - instClient.Keyring = client.Keyring - instClient.RepoURL = client.RepoURL - instClient.Version = client.Version - - r, err := runInstall(args, instClient, valueOpts, settings) - if err != nil { - return err - } - printReleaseInfo(r) - return nil - } else if err != nil { - return err - } - } - - chartPath, err := client.ChartPathOptions.LocateChart(args[1], settings) - if err != nil { - return err - } - - v, err := valueOpts.MergeValues(getter.All(settings)) - if err != nil { - return err - } - - // Check chart dependencies to make sure all are present in /charts - ch, err := helmLoader.Load(chartPath) - if err != nil { - return err - } - if req := ch.Metadata.Dependencies; req != nil { - if err := action.CheckDependencies(ch, req); err != nil { - return err - } - } - - if ch.Metadata.Deprecated { - logger.Log.Warningln("This chart is deprecated") - } - - r, err1 := client.Run(args[0], ch, v) - if err1 != nil { - return errors.Wrap(err1, "UPGRADE FAILED") - } - printReleaseInfo(r) - return nil -} - -func runInstall(args []string, client *action.Install, valueOpts *values.Options, settings *cli.EnvSettings) (*release.Release, error) { - if client.Version == "" && client.Devel { - client.Version = ">0.0.0-0" - } - - name, c, err := client.NameAndChart(args) - if err != nil { - return nil, err - } - client.ReleaseName = name - - cp, err := client.ChartPathOptions.LocateChart(c, settings) - if err != nil { - return nil, err - } - - p := getter.All(settings) - vals, err := valueOpts.MergeValues(p) - if err != nil { - return nil, err - } - // Check chart dependencies to make sure all are present in /charts - chartRequested, err := helmLoader.Load(cp) - if err != nil { - return nil, err - } - - if err := checkIfInstallable(chartRequested); err != nil { - return nil, err - } - - if chartRequested.Metadata.Deprecated { - logger.Log.Warningln("This chart is deprecated") - } - - if req := chartRequested.Metadata.Dependencies; req != nil { - if err := action.CheckDependencies(chartRequested, req); err != nil { - if client.DependencyUpdate { - man := &downloader.Manager{ - ChartPath: cp, - Keyring: client.ChartPathOptions.Keyring, - SkipUpdate: false, - Getters: p, - RepositoryConfig: settings.RepositoryConfig, - RepositoryCache: settings.RepositoryCache, - Debug: settings.Debug, - } - if err := man.Update(); err != nil { - return nil, err - } - // Reload the chart with the updated Chart.lock file. - if chartRequested, err = helmLoader.Load(cp); err != nil { - return nil, errors.Wrap(err, "failed reloading chart after repo update") - } - } else { - return nil, err - } - } - } - - return client.Run(chartRequested, vals) -} - -func checkIfInstallable(ch *chart.Chart) error { - switch ch.Metadata.Type { - case "", "application": - return nil - } - return errors.Errorf("%s charts are not installable", ch.Metadata.Type) -} - -func defaultKeyring() string { - if v, ok := os.LookupEnv("GNUPGHOME"); ok { - return filepath.Join(v, "pubring.gpg") - } - return filepath.Join(homedir.HomeDir(), ".gnupg", "pubring.gpg") -} - -func printReleaseInfo(release *release.Release) { - fmt.Printf("NAME: %s\n", release.Name) - if !release.Info.LastDeployed.IsZero() { - fmt.Printf("LAST DEPLOYED: %s\n", release.Info.LastDeployed.Format(time.ANSIC)) - } - fmt.Printf("NAMESPACE: %s\n", release.Namespace) - fmt.Printf("STATUS: %s\n", release.Info.Status.String()) - fmt.Printf("REVISION: %d\n", release.Version) -} diff --git a/cmd/kk/pkg/addons/manifests.go b/cmd/kk/pkg/addons/manifests.go deleted file mode 100644 index 82d163733..000000000 --- a/cmd/kk/pkg/addons/manifests.go +++ /dev/null @@ -1,200 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package addons - -import ( - "errors" - "fmt" - "os" - "path/filepath" - - "k8s.io/apimachinery/pkg/util/sets" - - versionutil "k8s.io/apimachinery/pkg/util/version" - "k8s.io/cli-runtime/pkg/genericclioptions" - "k8s.io/cli-runtime/pkg/printers" - "k8s.io/cli-runtime/pkg/resource" - "k8s.io/client-go/util/homedir" - "k8s.io/kubectl/pkg/cmd/apply" - cmdutil "k8s.io/kubectl/pkg/cmd/util" -) - -var ( - defaultCacheDir = filepath.Join(homedir.HomeDir(), ".kube", "cache") -) - -func InstallYaml(manifests []string, namespace, kubeConfig, version string) error { - - configFlags := NewConfigFlags(kubeConfig, namespace) - o, err := CreateApplyOptions(configFlags, manifests, version) - if err != nil { - return err - } - - if err := o.Run(); err != nil { - return err - } - - return nil -} - -func CreateApplyOptions(configFlags *genericclioptions.ConfigFlags, manifests []string, version string) (*apply.ApplyOptions, error) { - matchVersionKubeConfigFlags := NewMatchVersionFlags(configFlags) - f := cmdutil.NewFactory(matchVersionKubeConfigFlags) - ioStreams := genericclioptions.IOStreams{In: nil, Out: os.Stdout, ErrOut: os.Stderr} - - flags := apply.NewApplyFlags(f, ioStreams) - return ToOptions(flags, manifests, version) -} - -func ToOptions(flags *apply.ApplyFlags, manifests []string, version string) (*apply.ApplyOptions, error) { - serverSideApply := false - cmp, err := versionutil.MustParseSemantic(version).Compare("v1.16.0") - if err != nil { - return nil, errors.New(fmt.Sprintf("Failed to compare version: %v", err)) - } - if cmp == 0 || cmp == 1 { - serverSideApply = true - } else { - serverSideApply = false - } - - dryRunStrategy := cmdutil.DryRunNone - - dynamicClient, err := flags.Factory.DynamicClient() - if err != nil { - return nil, err - } - - dryRunVerifier := resource.NewQueryParamVerifier(dynamicClient, flags.Factory.OpenAPIGetter(), resource.QueryParamDryRun) - fieldValidationVerifier := resource.NewQueryParamVerifier(dynamicClient, flags.Factory.OpenAPIGetter(), resource.QueryParamFieldValidation) - fieldManager := "client-side-apply" - - // allow for a success message operation to be specified at print time - toPrinter := func(operation string) (printers.ResourcePrinter, error) { - flags.PrintFlags.NamePrintFlags.Operation = operation - cmdutil.PrintFlagsWithDryRunStrategy(flags.PrintFlags, dryRunStrategy) - return flags.PrintFlags.ToPrinter() - } - _ = flags.RecordFlags.CompleteWithChangeCause("") - - recorder, err := flags.RecordFlags.ToRecorder() - if err != nil { - return nil, err - } - - filenames := manifests - flags.DeleteFlags.FileNameFlags.Filenames = &filenames - - deleteOptions, err := flags.DeleteFlags.ToOptions(dynamicClient, flags.IOStreams) - if err != nil { - return nil, err - } - - err = deleteOptions.FilenameOptions.RequireFilenameOrKustomize() - if err != nil { - return nil, err - } - - openAPISchema, _ := flags.Factory.OpenAPISchema() - validator, err := flags.Factory.Validator("Ignore", fieldValidationVerifier) - if err != nil { - return nil, err - } - builder := flags.Factory.NewBuilder() - mapper, err := flags.Factory.ToRESTMapper() - if err != nil { - return nil, err - } - - namespace, enforceNamespace, err := flags.Factory.ToRawKubeConfigLoader().Namespace() - if err != nil { - return nil, err - } - - o := &apply.ApplyOptions{ - PrintFlags: flags.PrintFlags, - - DeleteOptions: deleteOptions, - ToPrinter: toPrinter, - ServerSideApply: serverSideApply, - ForceConflicts: true, - FieldManager: fieldManager, - Selector: flags.Selector, - DryRunStrategy: dryRunStrategy, - DryRunVerifier: dryRunVerifier, - Prune: flags.Prune, - PruneResources: flags.PruneResources, - All: flags.All, - Overwrite: flags.Overwrite, - OpenAPIPatch: flags.OpenAPIPatch, - PruneWhitelist: flags.PruneWhitelist, - - Recorder: recorder, - Namespace: namespace, - EnforceNamespace: enforceNamespace, - Validator: validator, - Builder: builder, - Mapper: mapper, - DynamicClient: dynamicClient, - OpenAPISchema: openAPISchema, - - IOStreams: flags.IOStreams, - - VisitedUids: sets.NewString(), - VisitedNamespaces: sets.NewString(), - } - - o.PostProcessorFn = o.PrintAndPrunePostProcessor() - - return o, nil -} - -func NewConfigFlags(kubeconfig, namespace string) *genericclioptions.ConfigFlags { - var impersonateGroup []string - insecure := false - - return &genericclioptions.ConfigFlags{ - Insecure: &insecure, - Timeout: stringptr("0"), - KubeConfig: stringptr(kubeconfig), - - CacheDir: stringptr(defaultCacheDir), - ClusterName: stringptr(""), - AuthInfoName: stringptr(""), - Context: stringptr(""), - Namespace: stringptr(namespace), - APIServer: stringptr(""), - TLSServerName: stringptr(""), - CertFile: stringptr(""), - KeyFile: stringptr(""), - CAFile: stringptr(""), - BearerToken: stringptr(""), - Impersonate: stringptr(""), - ImpersonateGroup: &impersonateGroup, - } -} - -func stringptr(val string) *string { - return &val -} - -func NewMatchVersionFlags(delegate genericclioptions.RESTClientGetter) *cmdutil.MatchVersionFlags { - return &cmdutil.MatchVersionFlags{ - Delegate: delegate, - } -} diff --git a/cmd/kk/pkg/addons/module.go b/cmd/kk/pkg/addons/module.go deleted file mode 100644 index ab833c23e..000000000 --- a/cmd/kk/pkg/addons/module.go +++ /dev/null @@ -1,46 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package addons - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" -) - -type AddonsModule struct { - common.KubeModule - Skip bool -} - -func (a *AddonsModule) IsSkip() bool { - return a.Skip -} - -func (a *AddonsModule) Init() { - a.Name = "AddonsModule" - a.Desc = "Install addons" - - install := &task.LocalTask{ - Name: "InstallAddons", - Desc: "Install addons", - Action: new(Install), - } - - a.Tasks = []task.Interface{ - install, - } -} diff --git a/cmd/kk/pkg/addons/tasks.go b/cmd/kk/pkg/addons/tasks.go deleted file mode 100644 index cd79d589f..000000000 --- a/cmd/kk/pkg/addons/tasks.go +++ /dev/null @@ -1,41 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package addons - -import ( - "fmt" - "path/filepath" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" -) - -type Install struct { - common.KubeAction -} - -func (i *Install) Execute(runtime connector.Runtime) error { - nums := len(i.KubeConf.Cluster.Addons) - for index, addon := range i.KubeConf.Cluster.Addons { - logger.Log.Messagef(runtime.RemoteHost().GetName(), "Install addon [%v-%v]: %s", nums, index, addon.Name) - if err := InstallAddons(i.KubeConf, &addon, filepath.Join(runtime.GetWorkDir(), fmt.Sprintf("config-%s", runtime.GetObjName()))); err != nil { - return err - } - } - return nil -} diff --git a/cmd/kk/pkg/artifact/manifest.go b/cmd/kk/pkg/artifact/manifest.go deleted file mode 100644 index a3f080030..000000000 --- a/cmd/kk/pkg/artifact/manifest.go +++ /dev/null @@ -1,206 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package artifact - -import ( - "bufio" - "context" - "fmt" - "os" - "sort" - "strings" - - mapset "github.com/deckarep/golang-set" - "github.com/pkg/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - versionutil "k8s.io/apimachinery/pkg/util/version" - - kubekeyv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/artifact/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/client/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -func CreateManifest(arg common.Argument, name string) error { - checkFileExists(arg.FilePath) - - client, err := kubernetes.NewClient(arg.KubeConfig) - if err != nil { - return errors.Wrap(err, "get kubernetes client failed") - } - - nodes, err := client.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) - if err != nil { - return err - } - - archSet := mapset.NewThreadUnsafeSet() - containerSet := mapset.NewThreadUnsafeSet() - imagesSet := mapset.NewThreadUnsafeSet() - osSet := mapset.NewThreadUnsafeSet() - - maxKubeletVersion := versionutil.MustParseGeneric("v0.0.0") - kubernetesDistribution := kubekeyv1alpha2.KubernetesDistribution{} - for _, node := range nodes.Items { - containerStrArr := strings.Split(node.Status.NodeInfo.ContainerRuntimeVersion, "://") - containerRuntime := kubekeyv1alpha2.ContainerRuntime{ - Type: containerStrArr[0], - Version: containerStrArr[1], - } - if containerRuntime.Type == "containerd" && - versionutil.MustParseSemantic(containerRuntime.Version).LessThan(versionutil.MustParseSemantic("1.6.2")) { - containerRuntime.Version = "1.6.2" - } - containerSet.Add(containerRuntime) - - archSet.Add(node.Status.NodeInfo.Architecture) - for _, image := range node.Status.Images { - for _, name := range image.Names { - if !strings.Contains(name, "@sha256") { - if containerRuntime.Type == kubekeyv1alpha2.Docker { - arr := strings.Split(name, "/") - switch len(arr) { - case 1: - name = fmt.Sprintf("docker.io/library/%s", name) - case 2: - name = fmt.Sprintf("docker.io/%s", name) - } - } - imagesSet.Add(name) - } - } - } - - // todo: for now, the cases only have ubuntu, centos. Ant it need to check all linux distribution. - var ( - id, version string - ) - osImageArr := strings.Split(node.Status.NodeInfo.OSImage, " ") - switch strings.ToLower(osImageArr[0]) { - case "ubuntu": - id = "ubuntu" - v := strings.Split(osImageArr[1], ".") - if len(v) >= 2 { - version = fmt.Sprintf("%s.%s", v[0], v[1]) - } - case "centos": - id = "centos" - version = osImageArr[2] - case "rhel": - id = "rhel" - version = osImageArr[2] - default: - id = strings.ToLower(osImageArr[0]) - version = "Can't get the os version. Please edit it manually." - } - - osObj := kubekeyv1alpha2.OperatingSystem{ - Arch: node.Status.NodeInfo.Architecture, - Type: node.Status.NodeInfo.OperatingSystem, - Id: id, - Version: version, - OsImage: node.Status.NodeInfo.OSImage, - } - osSet.Add(osObj) - - kubeletStrArr := strings.Split(node.Status.NodeInfo.KubeletVersion, "+") - kubeletVersion := kubeletStrArr[0] - distribution := "kubernetes" - if len(kubeletStrArr) == 2 { - distribution = "k3s" - } - if maxKubeletVersion.LessThan(versionutil.MustParseGeneric(kubeletVersion)) { - maxKubeletVersion = versionutil.MustParseGeneric(kubeletVersion) - kubernetesDistribution.Version = fmt.Sprintf("v%s", maxKubeletVersion.String()) - kubernetesDistribution.Type = distribution - } - - } - - archArr := make([]string, 0, archSet.Cardinality()) - for _, v := range archSet.ToSlice() { - arch := v.(string) - archArr = append(archArr, arch) - } - imageArr := make([]string, 0, imagesSet.Cardinality()) - for _, v := range imagesSet.ToSlice() { - image := v.(string) - imageArr = append(imageArr, image) - } - osArr := make([]kubekeyv1alpha2.OperatingSystem, 0, osSet.Cardinality()) - for _, v := range osSet.ToSlice() { - osObj := v.(kubekeyv1alpha2.OperatingSystem) - osArr = append(osArr, osObj) - } - containerArr := make([]kubekeyv1alpha2.ContainerRuntime, 0, containerSet.Cardinality()) - for _, v := range containerSet.ToSlice() { - container := v.(kubekeyv1alpha2.ContainerRuntime) - containerArr = append(containerArr, container) - } - - // todo: Whether it need to detect components version - sort.Strings(imageArr) - options := &templates.Options{ - Name: name, - Arches: archArr, - OperatingSystems: osArr, - KubernetesDistributions: []kubekeyv1alpha2.KubernetesDistribution{kubernetesDistribution}, - Components: kubekeyv1alpha2.Components{ - Helm: kubekeyv1alpha2.Helm{Version: kubekeyv1alpha2.DefaultHelmVersion}, - CNI: kubekeyv1alpha2.CNI{Version: kubekeyv1alpha2.DefaultCniVersion}, - ETCD: kubekeyv1alpha2.ETCD{Version: kubekeyv1alpha2.DefaultEtcdVersion}, - Crictl: kubekeyv1alpha2.Crictl{Version: kubekeyv1alpha2.DefaultCrictlVersion}, - Calicoctl: kubekeyv1alpha2.Calicoctl{Version: kubekeyv1alpha2.DefaultCalicoVersion}, - ContainerRuntimes: containerArr, - }, - Images: imageArr, - } - - manifestStr, err := templates.RenderManifest(options) - - if err := os.WriteFile(arg.FilePath, []byte(manifestStr), 0644); err != nil { - return errors.Wrap(err, fmt.Sprintf("write file %s failed", arg.FilePath)) - } - - fmt.Println("Generate KubeKey manifest file successfully") - return nil -} - -func checkFileExists(fileName string) { - if util.IsExist(fileName) { - reader := bufio.NewReader(os.Stdin) - stop := false - for { - if stop { - break - } - fmt.Printf("%s already exists. Are you sure you want to overwrite this file? [yes/no]: ", fileName) - input, _ := reader.ReadString('\n') - input = strings.TrimSpace(input) - - if input != "" { - switch input { - case "yes": - stop = true - case "no": - os.Exit(0) - } - } - } - } -} diff --git a/cmd/kk/pkg/artifact/module.go b/cmd/kk/pkg/artifact/module.go deleted file mode 100644 index d2de0e042..000000000 --- a/cmd/kk/pkg/artifact/module.go +++ /dev/null @@ -1,108 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package artifact - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" -) - -type RepositoryModule struct { - common.ArtifactModule -} - -func (r *RepositoryModule) Init() { - r.Name = "RepositoryModule" - r.Desc = "Get OS repository ISO file" - - download := &task.LocalTask{ - Name: "DownloadISOFile", - Desc: "Download iso file into work dir", - Prepare: new(EnableDownload), - Action: new(DownloadISOFile), - } - - localCopy := &task.LocalTask{ - Name: "LocalCopy", - Desc: "Copy local iso file into artifact dir", - Action: new(LocalCopy), - } - - r.Tasks = []task.Interface{ - download, - localCopy, - } -} - -type ArchiveModule struct { - common.ArtifactModule -} - -func (a *ArchiveModule) Init() { - a.Name = "ArtifactArchiveModule" - a.Desc = "Archive the dependencies" - - archive := &task.LocalTask{ - Name: "ArchiveDependencies", - Desc: "Archive the dependencies", - Action: new(ArchiveDependencies), - } - - a.Tasks = []task.Interface{ - archive, - } -} - -type UnArchiveModule struct { - common.KubeModule - Skip bool -} - -func (u *UnArchiveModule) IsSkip() bool { - return u.Skip -} - -func (u *UnArchiveModule) Init() { - u.Name = "UnArchiveArtifactModule" - u.Desc = "UnArchive the KubeKey artifact" - - md5Check := &task.LocalTask{ - Name: "CheckArtifactMd5", - Desc: "Check the KubeKey artifact md5 value", - Action: new(Md5Check), - } - - unArchive := &task.LocalTask{ - Name: "UnArchiveArtifact", - Desc: "UnArchive the KubeKey artifact", - Prepare: &Md5AreEqual{Not: true}, - Action: new(UnArchive), - } - - createMd5File := &task.LocalTask{ - Name: "CreateArtifactMd5File", - Desc: "Create the KubeKey artifact Md5 file", - Prepare: &Md5AreEqual{Not: true}, - Action: new(CreateMd5File), - } - - u.Tasks = []task.Interface{ - md5Check, - unArchive, - createMd5File, - } -} diff --git a/cmd/kk/pkg/artifact/prepares.go b/cmd/kk/pkg/artifact/prepares.go deleted file mode 100644 index 3759f11a9..000000000 --- a/cmd/kk/pkg/artifact/prepares.go +++ /dev/null @@ -1,54 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package artifact - -import ( - "fmt" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type EnableDownload struct { - common.ArtifactPrepare -} - -func (e *EnableDownload) PreCheck(_ connector.Runtime) (bool, error) { - for _, sys := range e.Manifest.Spec.OperatingSystems { - if sys.Repository.Iso.LocalPath == "" && sys.Repository.Iso.Url != "" { - return true, nil - } - } - return false, nil -} - -type Md5AreEqual struct { - common.KubePrepare - Not bool -} - -func (m *Md5AreEqual) PreCheck(_ connector.Runtime) (bool, error) { - equal, ok := m.ModuleCache.GetMustBool("md5AreEqual") - if !ok { - return false, fmt.Errorf("get md5 equal value from module cache failed") - } - - if equal { - return !m.Not, nil - } - return m.Not, nil -} diff --git a/cmd/kk/pkg/artifact/tasks.go b/cmd/kk/pkg/artifact/tasks.go deleted file mode 100644 index 89fb033bb..000000000 --- a/cmd/kk/pkg/artifact/tasks.go +++ /dev/null @@ -1,172 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package artifact - -import ( - "fmt" - "io" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - coreutil "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -type DownloadISOFile struct { - common.ArtifactAction -} - -func (d *DownloadISOFile) Execute(runtime connector.Runtime) error { - for i, sys := range d.Manifest.Spec.OperatingSystems { - if sys.Repository.Iso.Url == "" { - continue - } - - fileName := fmt.Sprintf("%s-%s-%s.iso", sys.Id, sys.Version, sys.Arch) - filePath := filepath.Join(runtime.GetWorkDir(), fileName) - getCmd := d.Manifest.Arg.DownloadCommand(filePath, sys.Repository.Iso.Url) - - cmd := exec.Command("/bin/sh", "-c", getCmd) - stdout, err := cmd.StdoutPipe() - if err != nil { - return fmt.Errorf("Failed to download %s iso file: %s error: %w ", fileName, getCmd, err) - } - cmd.Stderr = cmd.Stdout - - if err = cmd.Start(); err != nil { - return fmt.Errorf("Failed to download %s iso file: %s error: %w ", fileName, getCmd, err) - } - for { - tmp := make([]byte, 1024) - _, err := stdout.Read(tmp) - fmt.Print(string(tmp)) - if errors.Is(err, io.EOF) { - break - } else if err != nil { - logger.Log.Errorln(err) - break - } - } - if err = cmd.Wait(); err != nil { - return fmt.Errorf("Failed to download %s iso file: %s error: %w ", fileName, getCmd, err) - } - d.Manifest.Spec.OperatingSystems[i].Repository.Iso.LocalPath = filePath - } - return nil -} - -type LocalCopy struct { - common.ArtifactAction -} - -func (l *LocalCopy) Execute(runtime connector.Runtime) error { - for _, sys := range l.Manifest.Spec.OperatingSystems { - if sys.Repository.Iso.LocalPath == "" { - continue - } - - dir := filepath.Join(runtime.GetWorkDir(), common.Artifact, "repository", sys.Arch, sys.Id, sys.Version) - if err := coreutil.Mkdir(dir); err != nil { - return errors.Wrapf(errors.WithStack(err), "mkdir %s failed", dir) - } - - path := filepath.Join(dir, fmt.Sprintf("%s-%s-%s.iso", sys.Id, sys.Version, sys.Arch)) - if out, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("sudo cp -f %s %s", sys.Repository.Iso.LocalPath, path)).CombinedOutput(); err != nil { - return errors.Errorf("copy %s to %s failed: %s", sys.Repository.Iso.LocalPath, path, string(out)) - } - } - - return nil -} - -type ArchiveDependencies struct { - common.ArtifactAction -} - -func (a *ArchiveDependencies) Execute(runtime connector.Runtime) error { - src := filepath.Join(runtime.GetWorkDir(), common.Artifact) - if err := coreutil.Tar(src, a.Manifest.Arg.Output, src); err != nil { - return errors.Wrapf(errors.WithStack(err), "archive %s failed", src) - } - - // remove the src directory - if err := os.RemoveAll(src); err != nil { - return errors.Wrapf(errors.WithStack(err), "remove %s failed", src) - } - return nil -} - -type UnArchive struct { - common.KubeAction -} - -func (u *UnArchive) Execute(runtime connector.Runtime) error { - if err := coreutil.Untar(u.KubeConf.Arg.Artifact, runtime.GetWorkDir()); err != nil { - return errors.Wrapf(errors.WithStack(err), "unArchive %s failed", u.KubeConf.Arg.Artifact) - } - return nil -} - -type Md5Check struct { - common.KubeAction -} - -func (m *Md5Check) Execute(runtime connector.Runtime) error { - m.ModuleCache.Set("md5AreEqual", false) - - // check if there is a md5.sum file. This file's content contains the last artifact md5 value. - oldFile := filepath.Join(runtime.GetWorkDir(), "artifact.md5") - if exist := coreutil.IsExist(oldFile); !exist { - return nil - } - - oldMd5, err := os.ReadFile(oldFile) - if err != nil { - return errors.Wrapf(errors.WithStack(err), "read old md5 file %s failed", oldFile) - } - - newMd5 := coreutil.LocalMd5Sum(m.KubeConf.Arg.Artifact) - - if string(oldMd5) == newMd5 { - m.ModuleCache.Set("md5AreEqual", true) - } - return nil -} - -type CreateMd5File struct { - common.KubeAction -} - -func (c *CreateMd5File) Execute(runtime connector.Runtime) error { - oldFile := filepath.Join(runtime.GetWorkDir(), "artifact.md5") - newMd5 := coreutil.LocalMd5Sum(c.KubeConf.Arg.Artifact) - f, err := os.Create(oldFile) - if err != nil { - return errors.Wrapf(errors.WithStack(err), "create md5 fild %s failed", oldFile) - } - - if _, err := io.Copy(f, strings.NewReader(newMd5)); err != nil { - return errors.Wrapf(errors.WithStack(err), "write md5 value to file %s failed", oldFile) - } - return nil -} diff --git a/cmd/kk/pkg/artifact/templates/manifest.go b/cmd/kk/pkg/artifact/templates/manifest.go deleted file mode 100644 index 7b88d4fa7..000000000 --- a/cmd/kk/pkg/artifact/templates/manifest.go +++ /dev/null @@ -1,100 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" - - kubekeyv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -// Manifest defines the template of manifest file. -var Manifest = template.Must(template.New("Spec").Parse( - dedent.Dedent(` -apiVersion: kubekey.kubesphere.io/v1alpha2 -kind: Manifest -metadata: - name: {{ .Options.Name }} -spec: - arches: - {{- range .Options.Arches }} - - {{ . }} - {{- end }} - operatingSystems: - {{- range $i, $v := .Options.OperatingSystems }} - - arch: {{ $v.Arch }} - type: {{ $v.Type }} - id: {{ $v.Id }} - version: "{{ $v.Version }}" - osImage: {{ $v.OsImage }} - repository: - iso: - localPath: - url: - {{- end }} - kubernetesDistributions: - {{- range $i, $v := .Options.KubernetesDistributions }} - - type: {{ $v.Type }} - version: {{ $v.Version }} - {{- end}} - components: - helm: - version: {{ .Options.Components.Helm.Version }} - cni: - version: {{ .Options.Components.CNI.Version }} - etcd: - version: {{ .Options.Components.ETCD.Version }} - containerRuntimes: - {{- range $i, $v := .Options.Components.ContainerRuntimes }} - - type: {{ $v.Type }} - version: {{ $v.Version }} - {{- end}} - crictl: - version: {{ .Options.Components.Crictl.Version }} - ## - # docker-registry: - # version: "2" - # harbor: - # version: v2.4.1 - # docker-compose: - # version: v2.2.2 - images: - {{- range .Options.Images }} - - {{ . }} - {{- end }} - registry: - auths: {} - - `))) - -type Options struct { - Name string - Arches []string - OperatingSystems []kubekeyv1alpha2.OperatingSystem - KubernetesDistributions []kubekeyv1alpha2.KubernetesDistribution - Components kubekeyv1alpha2.Components - Images []string -} - -func RenderManifest(opt *Options) (string, error) { - return util.Render(Manifest, util.Data{ - "Options": opt, - }) -} diff --git a/cmd/kk/pkg/binaries/k3s.go b/cmd/kk/pkg/binaries/k3s.go deleted file mode 100644 index 0b6e9325d..000000000 --- a/cmd/kk/pkg/binaries/k3s.go +++ /dev/null @@ -1,129 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package binaries - -import ( - "fmt" - "os/exec" - - "github.com/pkg/errors" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" -) - -// K3sFilesDownloadHTTP defines the kubernetes' binaries that need to be downloaded in advance and downloads them. -func K3sFilesDownloadHTTP(kubeConf *common.KubeConf, path, version, arch string, pipelineCache *cache.Cache) error { - - etcd := files.NewKubeBinary("etcd", arch, kubekeyapiv1alpha2.DefaultEtcdVersion, path, kubeConf.Arg.DownloadCommand) - kubecni := files.NewKubeBinary("kubecni", arch, kubekeyapiv1alpha2.DefaultCniVersion, path, kubeConf.Arg.DownloadCommand) - helm := files.NewKubeBinary("helm", arch, kubekeyapiv1alpha2.DefaultHelmVersion, path, kubeConf.Arg.DownloadCommand) - k3s := files.NewKubeBinary("k3s", arch, version, path, kubeConf.Arg.DownloadCommand) - calicoctl := files.NewKubeBinary("calicoctl", arch, kubekeyapiv1alpha2.DefaultCalicoVersion, path, kubeConf.Arg.DownloadCommand) - - binaries := []*files.KubeBinary{k3s, helm, kubecni, etcd} - - if kubeConf.Cluster.Network.Plugin == "calico" { - binaries = append(binaries, calicoctl) - } - - binariesMap := make(map[string]*files.KubeBinary) - for _, binary := range binaries { - if err := binary.CreateBaseDir(); err != nil { - return errors.Wrapf(errors.WithStack(err), "create file %s base dir failed", binary.FileName) - } - - logger.Log.Messagef(common.LocalHost, "downloading %s %s %s ...", arch, binary.ID, binary.Version) - - binariesMap[binary.ID] = binary - if util.IsExist(binary.Path()) { - // download it again if it's incorrect - p := binary.Path() - if err := binary.SHA256Check(); err != nil { - _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", p)).Run() - } else { - continue - } - } - - if err := binary.Download(); err != nil { - return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.ID, binary.GetCmd(), err) - } - } - - pipelineCache.Set(common.KubeBinaries+"-"+arch, binariesMap) - return nil -} - -func K3sArtifactBinariesDownload(manifest *common.ArtifactManifest, path, arch, version string) error { - m := manifest.Spec - - etcd := files.NewKubeBinary("etcd", arch, m.Components.ETCD.Version, path, manifest.Arg.DownloadCommand) - kubecni := files.NewKubeBinary("kubecni", arch, m.Components.CNI.Version, path, manifest.Arg.DownloadCommand) - helm := files.NewKubeBinary("helm", arch, m.Components.Helm.Version, path, manifest.Arg.DownloadCommand) - k3s := files.NewKubeBinary("k3s", arch, version, path, manifest.Arg.DownloadCommand) - crictl := files.NewKubeBinary("crictl", arch, m.Components.Crictl.Version, path, manifest.Arg.DownloadCommand) - binaries := []*files.KubeBinary{k3s, helm, kubecni, etcd} - - dockerArr := make([]*files.KubeBinary, 0, 0) - dockerVersionMap := make(map[string]struct{}) - for _, c := range m.Components.ContainerRuntimes { - var dockerVersion string - if c.Type == common.Docker { - dockerVersion = c.Version - } else { - dockerVersion = kubekeyapiv1alpha2.DefaultDockerVersion - } - if _, ok := dockerVersionMap[dockerVersion]; !ok { - dockerVersionMap[dockerVersion] = struct{}{} - docker := files.NewKubeBinary("docker", arch, dockerVersion, path, manifest.Arg.DownloadCommand) - dockerArr = append(dockerArr, docker) - } - } - - binaries = append(binaries, dockerArr...) - if m.Components.Crictl.Version != "" { - binaries = append(binaries, crictl) - } - - for _, binary := range binaries { - if err := binary.CreateBaseDir(); err != nil { - return errors.Wrapf(errors.WithStack(err), "create file %s base dir failed", binary.FileName) - } - - logger.Log.Messagef(common.LocalHost, "downloading %s %s %s ...", arch, binary.ID, binary.Version) - - if util.IsExist(binary.Path()) { - // download it again if it's incorrect - if err := binary.SHA256Check(); err != nil { - _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", binary.Path())).Run() - } else { - continue - } - } - - if err := binary.Download(); err != nil { - return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.ID, binary.GetCmd(), err) - } - } - - return nil -} diff --git a/cmd/kk/pkg/binaries/k8e.go b/cmd/kk/pkg/binaries/k8e.go deleted file mode 100644 index 05b2eb61f..000000000 --- a/cmd/kk/pkg/binaries/k8e.go +++ /dev/null @@ -1,123 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package binaries - -import ( - "fmt" - "os/exec" - - "github.com/pkg/errors" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" -) - -// K8eFilesDownloadHTTP defines the kubernetes' binaries that need to be downloaded in advance and downloads them. -func K8eFilesDownloadHTTP(kubeConf *common.KubeConf, path, version, arch string, pipelineCache *cache.Cache) error { - - etcd := files.NewKubeBinary("etcd", arch, kubekeyapiv1alpha2.DefaultEtcdVersion, path, kubeConf.Arg.DownloadCommand) - kubecni := files.NewKubeBinary("kubecni", arch, kubekeyapiv1alpha2.DefaultCniVersion, path, kubeConf.Arg.DownloadCommand) - helm := files.NewKubeBinary("helm", arch, kubekeyapiv1alpha2.DefaultHelmVersion, path, kubeConf.Arg.DownloadCommand) - k8e := files.NewKubeBinary("k8e", arch, version, path, kubeConf.Arg.DownloadCommand) - - binaries := []*files.KubeBinary{k8e, helm, kubecni, etcd} - binariesMap := make(map[string]*files.KubeBinary) - for _, binary := range binaries { - if err := binary.CreateBaseDir(); err != nil { - return errors.Wrapf(errors.WithStack(err), "create file %s base dir failed", binary.FileName) - } - - logger.Log.Messagef(common.LocalHost, "downloading %s %s %s ...", arch, binary.ID, binary.Version) - - binariesMap[binary.ID] = binary - if util.IsExist(binary.Path()) { - // download it again if it's incorrect - p := binary.Path() - if err := binary.SHA256Check(); err != nil { - _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", p)).Run() - } else { - continue - } - } - - if err := binary.Download(); err != nil { - return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.ID, binary.GetCmd(), err) - } - } - - pipelineCache.Set(common.KubeBinaries+"-"+arch, binariesMap) - return nil -} - -func K8eArtifactBinariesDownload(manifest *common.ArtifactManifest, path, arch, version string) error { - m := manifest.Spec - - etcd := files.NewKubeBinary("etcd", arch, m.Components.ETCD.Version, path, manifest.Arg.DownloadCommand) - kubecni := files.NewKubeBinary("kubecni", arch, m.Components.CNI.Version, path, manifest.Arg.DownloadCommand) - helm := files.NewKubeBinary("helm", arch, m.Components.Helm.Version, path, manifest.Arg.DownloadCommand) - k8e := files.NewKubeBinary("k8e", arch, version, path, manifest.Arg.DownloadCommand) - crictl := files.NewKubeBinary("crictl", arch, m.Components.Crictl.Version, path, manifest.Arg.DownloadCommand) - binaries := []*files.KubeBinary{k8e, helm, kubecni, etcd} - - dockerArr := make([]*files.KubeBinary, 0, 0) - dockerVersionMap := make(map[string]struct{}) - for _, c := range m.Components.ContainerRuntimes { - var dockerVersion string - if c.Type == common.Docker { - dockerVersion = c.Version - } else { - dockerVersion = kubekeyapiv1alpha2.DefaultDockerVersion - } - if _, ok := dockerVersionMap[dockerVersion]; !ok { - dockerVersionMap[dockerVersion] = struct{}{} - docker := files.NewKubeBinary("docker", arch, dockerVersion, path, manifest.Arg.DownloadCommand) - dockerArr = append(dockerArr, docker) - } - } - - binaries = append(binaries, dockerArr...) - if m.Components.Crictl.Version != "" { - binaries = append(binaries, crictl) - } - - for _, binary := range binaries { - if err := binary.CreateBaseDir(); err != nil { - return errors.Wrapf(errors.WithStack(err), "create file %s base dir failed", binary.FileName) - } - - logger.Log.Messagef(common.LocalHost, "downloading %s %s %s ...", arch, binary.ID, binary.Version) - - if util.IsExist(binary.Path()) { - // download it again if it's incorrect - if err := binary.SHA256Check(); err != nil { - _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", binary.Path())).Run() - } else { - continue - } - } - - if err := binary.Download(); err != nil { - return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.ID, binary.GetCmd(), err) - } - } - - return nil -} diff --git a/cmd/kk/pkg/binaries/kubernetes.go b/cmd/kk/pkg/binaries/kubernetes.go deleted file mode 100644 index 4e381305a..000000000 --- a/cmd/kk/pkg/binaries/kubernetes.go +++ /dev/null @@ -1,199 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package binaries - -import ( - "fmt" - "os/exec" - - "github.com/pkg/errors" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" -) - -// K8sFilesDownloadHTTP defines the kubernetes' binaries that need to be downloaded in advance and downloads them. -func K8sFilesDownloadHTTP(kubeConf *common.KubeConf, path, version, arch string, pipelineCache *cache.Cache) error { - - etcd := files.NewKubeBinary("etcd", arch, kubekeyapiv1alpha2.DefaultEtcdVersion, path, kubeConf.Arg.DownloadCommand) - kubeadm := files.NewKubeBinary("kubeadm", arch, version, path, kubeConf.Arg.DownloadCommand) - kubelet := files.NewKubeBinary("kubelet", arch, version, path, kubeConf.Arg.DownloadCommand) - kubectl := files.NewKubeBinary("kubectl", arch, version, path, kubeConf.Arg.DownloadCommand) - kubecni := files.NewKubeBinary("kubecni", arch, kubekeyapiv1alpha2.DefaultCniVersion, path, kubeConf.Arg.DownloadCommand) - helm := files.NewKubeBinary("helm", arch, kubekeyapiv1alpha2.DefaultHelmVersion, path, kubeConf.Arg.DownloadCommand) - docker := files.NewKubeBinary("docker", arch, kubekeyapiv1alpha2.DefaultDockerVersion, path, kubeConf.Arg.DownloadCommand) - crictl := files.NewKubeBinary("crictl", arch, kubekeyapiv1alpha2.DefaultCrictlVersion, path, kubeConf.Arg.DownloadCommand) - containerd := files.NewKubeBinary("containerd", arch, kubekeyapiv1alpha2.DefaultContainerdVersion, path, kubeConf.Arg.DownloadCommand) - runc := files.NewKubeBinary("runc", arch, kubekeyapiv1alpha2.DefaultRuncVersion, path, kubeConf.Arg.DownloadCommand) - calicoctl := files.NewKubeBinary("calicoctl", arch, kubekeyapiv1alpha2.DefaultCalicoVersion, path, kubeConf.Arg.DownloadCommand) - - binaries := []*files.KubeBinary{kubeadm, kubelet, kubectl, helm, kubecni, crictl, etcd} - - if kubeConf.Cluster.Kubernetes.ContainerManager == kubekeyapiv1alpha2.Docker { - binaries = append(binaries, docker) - } else if kubeConf.Cluster.Kubernetes.ContainerManager == kubekeyapiv1alpha2.Containerd { - binaries = append(binaries, containerd, runc) - } - - if kubeConf.Cluster.Network.Plugin == "calico" { - binaries = append(binaries, calicoctl) - } - - binariesMap := make(map[string]*files.KubeBinary) - for _, binary := range binaries { - if err := binary.CreateBaseDir(); err != nil { - return errors.Wrapf(errors.WithStack(err), "create file %s base dir failed", binary.FileName) - } - - logger.Log.Messagef(common.LocalHost, "downloading %s %s %s ...", arch, binary.ID, binary.Version) - - binariesMap[binary.ID] = binary - if util.IsExist(binary.Path()) { - // download it again if it's incorrect - if err := binary.SHA256Check(); err != nil { - p := binary.Path() - _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", p)).Run() - } else { - logger.Log.Messagef(common.LocalHost, "%s exists", binary.ID) - continue - } - } - - if err := binary.Download(); err != nil { - return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.ID, binary.GetCmd(), err) - } - } - - if kubeConf.Cluster.KubeSphere.Version == "v2.1.1" { - logger.Log.Infoln(fmt.Sprintf("Downloading %s ...", "helm2")) - if util.IsExist(fmt.Sprintf("%s/helm2", helm.BaseDir)) == false { - cmd := kubeConf.Arg.DownloadCommand(fmt.Sprintf("%s/helm2", helm.BaseDir), - fmt.Sprintf("https://kubernetes-helm.pek3b.qingstor.com/linux-%s/%s/helm", helm.Arch, "v2.16.9")) - if output, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput(); err != nil { - fmt.Println(string(output)) - return errors.Wrap(err, "Failed to download helm2 binary") - } - } - } - - pipelineCache.Set(common.KubeBinaries+"-"+arch, binariesMap) - return nil -} - -func KubernetesArtifactBinariesDownload(manifest *common.ArtifactManifest, path, arch, k8sVersion string) error { - m := manifest.Spec - - etcd := files.NewKubeBinary("etcd", arch, m.Components.ETCD.Version, path, manifest.Arg.DownloadCommand) - kubeadm := files.NewKubeBinary("kubeadm", arch, k8sVersion, path, manifest.Arg.DownloadCommand) - kubelet := files.NewKubeBinary("kubelet", arch, k8sVersion, path, manifest.Arg.DownloadCommand) - kubectl := files.NewKubeBinary("kubectl", arch, k8sVersion, path, manifest.Arg.DownloadCommand) - kubecni := files.NewKubeBinary("kubecni", arch, m.Components.CNI.Version, path, manifest.Arg.DownloadCommand) - helm := files.NewKubeBinary("helm", arch, m.Components.Helm.Version, path, manifest.Arg.DownloadCommand) - crictl := files.NewKubeBinary("crictl", arch, m.Components.Crictl.Version, path, manifest.Arg.DownloadCommand) - calicoctl := files.NewKubeBinary("calicoctl", arch, m.Components.Calicoctl.Version, path, manifest.Arg.DownloadCommand) - binaries := []*files.KubeBinary{kubeadm, kubelet, kubectl, helm, kubecni, etcd, calicoctl} - - containerManagerArr := make([]*files.KubeBinary, 0, 0) - containerManagerVersion := make(map[string]struct{}) - for _, c := range m.Components.ContainerRuntimes { - if _, ok := containerManagerVersion[c.Type+c.Version]; !ok { - containerManagerVersion[c.Type+c.Version] = struct{}{} - containerManager := files.NewKubeBinary(c.Type, arch, c.Version, path, manifest.Arg.DownloadCommand) - containerManagerArr = append(containerManagerArr, containerManager) - if c.Type == "containerd" { - runc := files.NewKubeBinary("runc", arch, kubekeyapiv1alpha2.DefaultRuncVersion, path, manifest.Arg.DownloadCommand) - containerManagerArr = append(containerManagerArr, runc) - } - } - } - - binaries = append(binaries, containerManagerArr...) - if m.Components.Crictl.Version != "" { - binaries = append(binaries, crictl) - } - - for _, binary := range binaries { - if err := binary.CreateBaseDir(); err != nil { - return errors.Wrapf(errors.WithStack(err), "create file %s base dir failed", binary.FileName) - } - - logger.Log.Messagef(common.LocalHost, "downloading %s %s %s ...", arch, binary.ID, binary.Version) - - if util.IsExist(binary.Path()) { - // download it again if it's incorrect - if err := binary.SHA256Check(); err != nil { - _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", binary.Path())).Run() - } else { - continue - } - } - - if err := binary.Download(); err != nil { - return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.ID, binary.GetCmd(), err) - } - } - - return nil -} - -// CriDownloadHTTP defines the kubernetes' binaries that need to be downloaded in advance and downloads them. -func CriDownloadHTTP(kubeConf *common.KubeConf, path, arch string, pipelineCache *cache.Cache) error { - - binaries := []*files.KubeBinary{} - switch kubeConf.Arg.Type { - case common.Docker: - docker := files.NewKubeBinary("docker", arch, kubekeyapiv1alpha2.DefaultDockerVersion, path, kubeConf.Arg.DownloadCommand) - binaries = append(binaries, docker) - case common.Containerd: - containerd := files.NewKubeBinary("containerd", arch, kubekeyapiv1alpha2.DefaultContainerdVersion, path, kubeConf.Arg.DownloadCommand) - runc := files.NewKubeBinary("runc", arch, kubekeyapiv1alpha2.DefaultRuncVersion, path, kubeConf.Arg.DownloadCommand) - crictl := files.NewKubeBinary("crictl", arch, kubekeyapiv1alpha2.DefaultCrictlVersion, path, kubeConf.Arg.DownloadCommand) - binaries = append(binaries, containerd, runc, crictl) - default: - } - binariesMap := make(map[string]*files.KubeBinary) - for _, binary := range binaries { - if err := binary.CreateBaseDir(); err != nil { - return errors.Wrapf(errors.WithStack(err), "create file %s base dir failed", binary.FileName) - } - - logger.Log.Messagef(common.LocalHost, "downloading %s %s %s ...", arch, binary.ID, binary.Version) - - binariesMap[binary.ID] = binary - if util.IsExist(binary.Path()) { - // download it again if it's incorrect - if err := binary.SHA256Check(); err != nil { - p := binary.Path() - _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", p)).Run() - } else { - logger.Log.Messagef(common.LocalHost, "%s exists", binary.ID) - continue - } - } - - if err := binary.Download(); err != nil { - return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.ID, binary.GetCmd(), err) - } - } - - pipelineCache.Set(common.KubeBinaries+"-"+arch, binariesMap) - return nil -} diff --git a/cmd/kk/pkg/binaries/module.go b/cmd/kk/pkg/binaries/module.go deleted file mode 100644 index 6facc57f2..000000000 --- a/cmd/kk/pkg/binaries/module.go +++ /dev/null @@ -1,193 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package binaries - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" -) - -type NodeBinariesModule struct { - common.KubeModule -} - -func (n *NodeBinariesModule) Init() { - n.Name = "NodeBinariesModule" - n.Desc = "Download installation binaries" - - download := &task.LocalTask{ - Name: "DownloadBinaries", - Desc: "Download installation binaries", - Action: new(Download), - } - - n.Tasks = []task.Interface{ - download, - } -} - -type K3sNodeBinariesModule struct { - common.KubeModule -} - -func (k *K3sNodeBinariesModule) Init() { - k.Name = "K3sNodeBinariesModule" - k.Desc = "Download installation binaries" - - download := &task.LocalTask{ - Name: "DownloadBinaries", - Desc: "Download installation binaries", - Action: new(K3sDownload), - } - - k.Tasks = []task.Interface{ - download, - } -} - -type K8eNodeBinariesModule struct { - common.KubeModule -} - -func (k *K8eNodeBinariesModule) Init() { - k.Name = "K8eNodeBinariesModule" - k.Desc = "Download installation binaries" - - download := &task.LocalTask{ - Name: "DownloadBinaries", - Desc: "Download installation binaries", - Action: new(K8eDownload), - } - - k.Tasks = []task.Interface{ - download, - } -} - -type ArtifactBinariesModule struct { - common.ArtifactModule -} - -func (a *ArtifactBinariesModule) Init() { - a.Name = "ArtifactBinariesModule" - a.Desc = "Download artifact binaries" - - download := &task.LocalTask{ - Name: "DownloadBinaries", - Desc: "Download manifest expect binaries", - Action: new(ArtifactDownload), - } - - a.Tasks = []task.Interface{ - download, - } -} - -type K3sArtifactBinariesModule struct { - common.ArtifactModule -} - -func (a *K3sArtifactBinariesModule) Init() { - a.Name = "K3sArtifactBinariesModule" - a.Desc = "Download artifact binaries" - - download := &task.LocalTask{ - Name: "K3sDownloadBinaries", - Desc: "Download k3s manifest expect binaries", - Action: new(K3sArtifactDownload), - } - - a.Tasks = []task.Interface{ - download, - } -} - -type K8eArtifactBinariesModule struct { - common.ArtifactModule -} - -func (a *K8eArtifactBinariesModule) Init() { - a.Name = "K8eArtifactBinariesModule" - a.Desc = "Download artifact binaries" - - download := &task.LocalTask{ - Name: "K8eDownloadBinaries", - Desc: "Download k8e manifest expect binaries", - Action: new(K8eArtifactDownload), - } - - a.Tasks = []task.Interface{ - download, - } -} - -type RegistryPackageModule struct { - common.KubeModule -} - -func (n *RegistryPackageModule) Init() { - n.Name = "RegistryPackageModule" - n.Desc = "Download registry package" - - if len(n.Runtime.GetHostsByRole(common.Registry)) == 0 { - logger.Log.Fatal(errors.New("[registry] node not found in the roleGroups of the configuration file")) - } - - download := &task.LocalTask{ - Name: "DownloadRegistryPackage", - Desc: "Download registry package", - Action: new(RegistryPackageDownload), - } - - n.Tasks = []task.Interface{ - download, - } -} - -type CriBinariesModule struct { - common.KubeModule -} - -func (i *CriBinariesModule) Init() { - i.Name = "CriBinariesModule" - i.Desc = "Download Cri package" - switch i.KubeConf.Arg.Type { - case common.Docker: - i.Tasks = CriBinaries(i) - case common.Containerd: - i.Tasks = CriBinaries(i) - default: - } - -} - -func CriBinaries(p *CriBinariesModule) []task.Interface { - - download := &task.LocalTask{ - Name: "DownloadCriPackage", - Desc: "Download Cri package", - Action: new(CriDownload), - } - - p.Tasks = []task.Interface{ - download, - } - return p.Tasks -} diff --git a/cmd/kk/pkg/binaries/registry.go b/cmd/kk/pkg/binaries/registry.go deleted file mode 100644 index ec31028a2..000000000 --- a/cmd/kk/pkg/binaries/registry.go +++ /dev/null @@ -1,126 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package binaries - -import ( - "fmt" - "os/exec" - - "github.com/pkg/errors" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" -) - -// RegistryPackageDownloadHTTP defines the kubernetes' binaries that need to be downloaded in advance and downloads them. -func RegistryPackageDownloadHTTP(kubeConf *common.KubeConf, path, arch string, pipelineCache *cache.Cache) error { - var binaries []*files.KubeBinary - - switch kubeConf.Cluster.Registry.Type { - case common.Harbor: - // TODO: Harbor only supports amd64, so there is no need to consider other architectures at present. - harbor := files.NewKubeBinary("harbor", arch, kubekeyapiv1alpha2.DefaultHarborVersion, path, kubeConf.Arg.DownloadCommand) - compose := files.NewKubeBinary("compose", arch, kubekeyapiv1alpha2.DefaultDockerComposeVersion, path, kubeConf.Arg.DownloadCommand) - docker := files.NewKubeBinary("docker", arch, kubekeyapiv1alpha2.DefaultDockerVersion, path, kubeConf.Arg.DownloadCommand) - binaries = []*files.KubeBinary{harbor, docker, compose} - default: - registry := files.NewKubeBinary("registry", arch, kubekeyapiv1alpha2.DefaultRegistryVersion, path, kubeConf.Arg.DownloadCommand) - binaries = []*files.KubeBinary{registry} - } - - binariesMap := make(map[string]*files.KubeBinary) - for _, binary := range binaries { - if err := binary.CreateBaseDir(); err != nil { - return errors.Wrapf(errors.WithStack(err), "create file %s base dir failed", binary.FileName) - } - - logger.Log.Messagef(common.LocalHost, "downloading %s %s %s ...", arch, binary.ID, binary.Version) - binariesMap[binary.ID] = binary - if util.IsExist(binary.Path()) { - // download it again if it's incorrect - if err := binary.SHA256Check(); err != nil { - p := binary.Path() - _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", p)).Run() - } else { - continue - } - } - - if err := binary.Download(); err != nil { - return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.ID, binary.GetCmd(), err) - } - } - - pipelineCache.Set(common.KubeBinaries+"-"+arch, binariesMap) - return nil -} - -func RegistryBinariesDownload(manifest *common.ArtifactManifest, path, arch string) error { - - m := manifest.Spec - binaries := make([]*files.KubeBinary, 0, 0) - - if m.Components.DockerRegistry.Version != "" { - registry := files.NewKubeBinary("registry", arch, kubekeyapiv1alpha2.DefaultRegistryVersion, path, manifest.Arg.DownloadCommand) - binaries = append(binaries, registry) - } - - if m.Components.Harbor.Version != "" { - harbor := files.NewKubeBinary("harbor", arch, kubekeyapiv1alpha2.DefaultHarborVersion, path, manifest.Arg.DownloadCommand) - if arch == "amd64" { - binaries = append(binaries, harbor) - } else { - logger.Log.Warningf("Harbor only supports amd64, the KubeKey artifact will not contain the Harbor %s", arch) - } - } - - if m.Components.DockerCompose.Version != "" { - compose := files.NewKubeBinary("compose", arch, kubekeyapiv1alpha2.DefaultDockerComposeVersion, path, manifest.Arg.DownloadCommand) - containerManager := files.NewKubeBinary("docker", arch, kubekeyapiv1alpha2.DefaultDockerVersion, path, manifest.Arg.DownloadCommand) - if arch == "amd64" { - binaries = append(binaries, compose) - binaries = append(binaries, containerManager) - } - } - - for _, binary := range binaries { - if err := binary.CreateBaseDir(); err != nil { - return errors.Wrapf(errors.WithStack(err), "create file %s base dir failed", binary.FileName) - } - - logger.Log.Messagef(common.LocalHost, "downloading %s %s %s ...", arch, binary.ID, binary.Version) - - if util.IsExist(binary.Path()) { - // download it again if it's incorrect - if err := binary.SHA256Check(); err != nil { - p := binary.Path() - _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", p)).Run() - } else { - continue - } - } - - if err := binary.Download(); err != nil { - return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.ID, binary.GetCmd(), err) - } - } - return nil -} diff --git a/cmd/kk/pkg/binaries/tasks.go b/cmd/kk/pkg/binaries/tasks.go deleted file mode 100644 index 7d829cb79..000000000 --- a/cmd/kk/pkg/binaries/tasks.go +++ /dev/null @@ -1,305 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package binaries - -import ( - "fmt" - "path/filepath" - - mapset "github.com/deckarep/golang-set" - "github.com/pkg/errors" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type Download struct { - common.KubeAction -} - -func (d *Download) Execute(runtime connector.Runtime) error { - cfg := d.KubeConf.Cluster - - var kubeVersion string - if cfg.Kubernetes.Version == "" { - kubeVersion = kubekeyapiv1alpha2.DefaultKubeVersion - } else { - kubeVersion = cfg.Kubernetes.Version - } - - archMap := make(map[string]bool) - for _, host := range cfg.Hosts { - switch host.Arch { - case "amd64": - archMap["amd64"] = true - case "arm64": - archMap["arm64"] = true - default: - return errors.New(fmt.Sprintf("Unsupported architecture: %s", host.Arch)) - } - } - - for arch := range archMap { - if err := K8sFilesDownloadHTTP(d.KubeConf, runtime.GetWorkDir(), kubeVersion, arch, d.PipelineCache); err != nil { - return err - } - } - return nil -} - -type K3sDownload struct { - common.KubeAction -} - -func (k *K3sDownload) Execute(runtime connector.Runtime) error { - cfg := k.KubeConf.Cluster - - var kubeVersion string - if cfg.Kubernetes.Version == "" { - kubeVersion = kubekeyapiv1alpha2.DefaultKubeVersion - } else { - kubeVersion = cfg.Kubernetes.Version - } - - archMap := make(map[string]bool) - for _, host := range cfg.Hosts { - switch host.Arch { - case "amd64": - archMap["amd64"] = true - case "arm64": - archMap["arm64"] = true - default: - return errors.New(fmt.Sprintf("Unsupported architecture: %s", host.Arch)) - } - } - - for arch := range archMap { - if err := K3sFilesDownloadHTTP(k.KubeConf, runtime.GetWorkDir(), kubeVersion, arch, k.PipelineCache); err != nil { - return err - } - } - return nil -} - -type K8eDownload struct { - common.KubeAction -} - -func (k *K8eDownload) Execute(runtime connector.Runtime) error { - cfg := k.KubeConf.Cluster - - var kubeVersion string - if cfg.Kubernetes.Version == "" { - kubeVersion = kubekeyapiv1alpha2.DefaultKubeVersion - } else { - kubeVersion = cfg.Kubernetes.Version - } - - archMap := make(map[string]bool) - for _, host := range cfg.Hosts { - switch host.Arch { - case "amd64": - archMap["amd64"] = true - case "arm64": - archMap["arm64"] = true - default: - return errors.New(fmt.Sprintf("Unsupported architecture: %s", host.Arch)) - } - } - - for arch := range archMap { - if err := K8eFilesDownloadHTTP(k.KubeConf, runtime.GetWorkDir(), kubeVersion, arch, k.PipelineCache); err != nil { - return err - } - } - return nil -} - -type ArtifactDownload struct { - common.ArtifactAction -} - -func (a *ArtifactDownload) Execute(runtime connector.Runtime) error { - manifest := a.Manifest.Spec - - archMap := make(map[string]bool) - for _, arch := range manifest.Arches { - switch arch { - case "amd64": - archMap["amd64"] = true - case "arm64": - archMap["arm64"] = true - default: - return errors.New(fmt.Sprintf("Unsupported architecture: %s", arch)) - } - } - - kubernetesSet := mapset.NewThreadUnsafeSet() - for _, k := range manifest.KubernetesDistributions { - kubernetesSet.Add(k) - } - - kubernetesVersions := make([]string, 0, kubernetesSet.Cardinality()) - for _, k := range kubernetesSet.ToSlice() { - k8s := k.(kubekeyapiv1alpha2.KubernetesDistribution) - kubernetesVersions = append(kubernetesVersions, k8s.Version) - } - - basePath := filepath.Join(runtime.GetWorkDir(), common.Artifact) - for arch := range archMap { - for _, version := range kubernetesVersions { - if err := KubernetesArtifactBinariesDownload(a.Manifest, basePath, arch, version); err != nil { - return err - } - } - - if err := RegistryBinariesDownload(a.Manifest, basePath, arch); err != nil { - return err - } - } - return nil -} - -type K3sArtifactDownload struct { - common.ArtifactAction -} - -func (a *K3sArtifactDownload) Execute(runtime connector.Runtime) error { - manifest := a.Manifest.Spec - - archMap := make(map[string]bool) - for _, arch := range manifest.Arches { - switch arch { - case "amd64": - archMap["amd64"] = true - case "arm64": - archMap["arm64"] = true - default: - return errors.New(fmt.Sprintf("Unsupported architecture: %s", arch)) - } - } - - kubernetesSet := mapset.NewThreadUnsafeSet() - for _, k := range manifest.KubernetesDistributions { - kubernetesSet.Add(k) - } - - kubernetesVersions := make([]string, 0, kubernetesSet.Cardinality()) - for _, k := range kubernetesSet.ToSlice() { - k8s := k.(kubekeyapiv1alpha2.KubernetesDistribution) - kubernetesVersions = append(kubernetesVersions, k8s.Version) - } - - basePath := filepath.Join(runtime.GetWorkDir(), common.Artifact) - for arch := range archMap { - for _, version := range kubernetesVersions { - if err := K3sArtifactBinariesDownload(a.Manifest, basePath, arch, version); err != nil { - return err - } - } - - if err := RegistryBinariesDownload(a.Manifest, basePath, arch); err != nil { - return err - } - } - return nil -} - -type K8eArtifactDownload struct { - common.ArtifactAction -} - -func (a *K8eArtifactDownload) Execute(runtime connector.Runtime) error { - manifest := a.Manifest.Spec - - archMap := make(map[string]bool) - for _, arch := range manifest.Arches { - switch arch { - case "amd64": - archMap["amd64"] = true - case "arm64": - archMap["arm64"] = true - default: - return errors.New(fmt.Sprintf("Unsupported architecture: %s", arch)) - } - } - - kubernetesSet := mapset.NewThreadUnsafeSet() - for _, k := range manifest.KubernetesDistributions { - kubernetesSet.Add(k) - } - - kubernetesVersions := make([]string, 0, kubernetesSet.Cardinality()) - for _, k := range kubernetesSet.ToSlice() { - k8s := k.(kubekeyapiv1alpha2.KubernetesDistribution) - kubernetesVersions = append(kubernetesVersions, k8s.Version) - } - - basePath := filepath.Join(runtime.GetWorkDir(), common.Artifact) - for arch := range archMap { - for _, version := range kubernetesVersions { - if err := K8eArtifactBinariesDownload(a.Manifest, basePath, arch, version); err != nil { - return err - } - } - - if err := RegistryBinariesDownload(a.Manifest, basePath, arch); err != nil { - return err - } - } - return nil -} - -type RegistryPackageDownload struct { - common.KubeAction -} - -func (k *RegistryPackageDownload) Execute(runtime connector.Runtime) error { - arch := runtime.GetHostsByRole(common.Registry)[0].GetArch() - if err := RegistryPackageDownloadHTTP(k.KubeConf, runtime.GetWorkDir(), arch, k.PipelineCache); err != nil { - return err - } - - return nil -} - -type CriDownload struct { - common.KubeAction -} - -func (d *CriDownload) Execute(runtime connector.Runtime) error { - cfg := d.KubeConf.Cluster - archMap := make(map[string]bool) - for _, host := range cfg.Hosts { - switch host.Arch { - case "amd64": - archMap["amd64"] = true - case "arm64": - archMap["arm64"] = true - default: - return errors.New(fmt.Sprintf("Unsupported architecture: %s", host.Arch)) - } - } - - for arch := range archMap { - if err := CriDownloadHTTP(d.KubeConf, runtime.GetWorkDir(), arch, d.PipelineCache); err != nil { - return err - } - } - return nil -} diff --git a/cmd/kk/pkg/bootstrap/confirm/module.go b/cmd/kk/pkg/bootstrap/confirm/module.go deleted file mode 100644 index 5bb67d651..000000000 --- a/cmd/kk/pkg/bootstrap/confirm/module.go +++ /dev/null @@ -1,159 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package confirm - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" -) - -type InstallConfirmModule struct { - common.KubeModule - Skip bool -} - -func (i *InstallConfirmModule) IsSkip() bool { - return i.Skip -} - -func (i *InstallConfirmModule) Init() { - i.Name = "ConfirmModule" - i.Desc = "Display confirmation form" - - display := &task.LocalTask{ - Name: "ConfirmForm", - Desc: "Display confirmation form", - Action: new(InstallationConfirm), - } - - i.Tasks = []task.Interface{ - display, - } -} - -type DeleteClusterConfirmModule struct { - common.KubeModule - Skip bool -} - -func (d *DeleteClusterConfirmModule) IsSkip() bool { - return d.Skip -} - -func (d *DeleteClusterConfirmModule) Init() { - d.Name = "DeleteClusterConfirmModule" - d.Desc = "Display delete confirmation form" - - display := &task.LocalTask{ - Name: "ConfirmForm", - Desc: "Display confirmation form", - Action: &DeleteConfirm{Content: "cluster"}, - } - - d.Tasks = []task.Interface{ - display, - } -} - -type DeleteNodeConfirmModule struct { - common.KubeModule - Skip bool -} - -func (d *DeleteNodeConfirmModule) IsSkip() bool { - return d.Skip -} - -func (d *DeleteNodeConfirmModule) Init() { - d.Name = "DeleteNodeConfirmModule" - d.Desc = "Display delete node confirmation form" - - display := &task.LocalTask{ - Name: "ConfirmForm", - Desc: "Display confirmation form", - Action: &DeleteConfirm{Content: "node"}, - } - - d.Tasks = []task.Interface{ - display, - } -} - -type UpgradeConfirmModule struct { - common.KubeModule - Skip bool -} - -func (u *UpgradeConfirmModule) IsSkip() bool { - return u.Skip -} - -func (u *UpgradeConfirmModule) Init() { - u.Name = "UpgradeConfirmModule" - u.Desc = "Display upgrade confirmation form" - - display := &task.LocalTask{ - Name: "ConfirmForm", - Desc: "Display confirmation form", - Action: new(UpgradeConfirm), - } - - u.Tasks = []task.Interface{ - display, - } -} - -type CheckFileExistModule struct { - module.BaseTaskModule - FileName string -} - -func (c *CheckFileExistModule) Init() { - c.Name = "CheckFileExist" - c.Desc = "Check file if is existed" - - check := &task.LocalTask{ - Name: "CheckExist", - Desc: "Check output file if existed", - Action: &CheckFile{FileName: c.FileName}, - } - - c.Tasks = []task.Interface{ - check, - } -} - -type MigrateCriConfirmModule struct { - common.KubeModule -} - -func (d *MigrateCriConfirmModule) Init() { - d.Name = "MigrateCriConfirmModule" - d.Desc = "Display Migrate Cri form" - - display := &task.LocalTask{ - Name: "ConfirmForm", - Desc: "Display confirmation form", - Action: &MigrateCri{}, - } - - d.Tasks = []task.Interface{ - display, - } - -} diff --git a/cmd/kk/pkg/bootstrap/confirm/tasks.go b/cmd/kk/pkg/bootstrap/confirm/tasks.go deleted file mode 100644 index 06926da07..000000000 --- a/cmd/kk/pkg/bootstrap/confirm/tasks.go +++ /dev/null @@ -1,374 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package confirm - -import ( - "bufio" - "fmt" - "os" - "regexp" - "strings" - - "github.com/mitchellh/mapstructure" - "github.com/modood/table" - "github.com/pkg/errors" - versionutil "k8s.io/apimachinery/pkg/util/version" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -// PreCheckResults defines the items to be checked. -type PreCheckResults struct { - Name string `table:"name"` - Sudo string `table:"sudo"` - Curl string `table:"curl"` - Openssl string `table:"openssl"` - Ebtables string `table:"ebtables"` - Socat string `table:"socat"` - Ipset string `table:"ipset"` - Ipvsadm string `table:"ipvsadm"` - Conntrack string `table:"conntrack"` - Chronyd string `table:"chrony"` - Docker string `table:"docker"` - Containerd string `table:"containerd"` - Nfs string `table:"nfs client"` - Ceph string `table:"ceph client"` - Glusterfs string `table:"glusterfs client"` - Time string `table:"time"` -} - -type InstallationConfirm struct { - common.KubeAction -} - -func (i *InstallationConfirm) Execute(runtime connector.Runtime) error { - var ( - results []PreCheckResults - stopFlag bool - ) - - pre := make([]map[string]string, 0, len(runtime.GetAllHosts())) - for _, host := range runtime.GetAllHosts() { - if v, ok := host.GetCache().Get(common.NodePreCheck); ok { - pre = append(pre, v.(map[string]string)) - } else { - return errors.New("get node check result failed by host cache") - } - } - - for node := range pre { - var result PreCheckResults - _ = mapstructure.Decode(pre[node], &result) - results = append(results, result) - } - table.OutputA(results) - reader := bufio.NewReader(os.Stdin) - - if i.KubeConf.Arg.Artifact == "" { - for _, host := range results { - if host.Sudo == "" { - logger.Log.Errorf("%s: sudo is required.", host.Name) - stopFlag = true - } - - if host.Conntrack == "" { - logger.Log.Errorf("%s: conntrack is required.", host.Name) - stopFlag = true - } - - if host.Socat == "" { - logger.Log.Errorf("%s: socat is required.", host.Name) - stopFlag = true - } - } - } - - fmt.Println("") - fmt.Println("This is a simple check of your environment.") - fmt.Println("Before installation, ensure that your machines meet all requirements specified at") - fmt.Println("https://github.com/kubesphere/kubekey#requirements-and-recommendations") - fmt.Println("") - - if k8sVersion, err := versionutil.ParseGeneric(i.KubeConf.Cluster.Kubernetes.Version); err == nil { - if k8sVersion.AtLeast(versionutil.MustParseSemantic("v1.24.0")) && i.KubeConf.Cluster.Kubernetes.ContainerManager == common.Docker { - fmt.Println("[Notice]") - fmt.Println("Incorrect runtime. Please specify a container runtime other than Docker to install Kubernetes v1.24 or later.") - fmt.Println("You can set \"spec.kubernetes.containerManager\" in the configuration file to \"containerd\" or add \"--container-manager containerd\" to the \"./kk create cluster\" command.") - fmt.Println("For more information, see:") - fmt.Println("https://github.com/kubesphere/kubekey/blob/master/docs/commands/kk-create-cluster.md") - fmt.Println("https://kubernetes.io/docs/setup/production-environment/container-runtimes/#container-runtimes") - fmt.Println("https://kubernetes.io/blog/2022/02/17/dockershim-faq/") - fmt.Println("") - stopFlag = true - } - } - - if stopFlag { - os.Exit(1) - } - - if i.KubeConf.Arg.SkipConfirmCheck { - return nil - } - - confirmOK := false - for !confirmOK { - fmt.Printf("Continue this installation? [yes/no]: ") - input, err := reader.ReadString('\n') - if err != nil { - logger.Log.Fatal(err) - } - input = strings.TrimSpace(strings.ToLower(input)) - - switch strings.ToLower(input) { - case "yes", "y": - confirmOK = true - case "no", "n": - os.Exit(0) - default: - continue - } - } - return nil -} - -type DeleteConfirm struct { - common.KubeAction - Content string -} - -func (d *DeleteConfirm) Execute(runtime connector.Runtime) error { - reader := bufio.NewReader(os.Stdin) - - confirmOK := false - for !confirmOK { - fmt.Printf("Are you sure to delete this %s? [yes/no]: ", d.Content) - input, err := reader.ReadString('\n') - if err != nil { - return err - } - input = strings.ToLower(strings.TrimSpace(input)) - - switch strings.ToLower(input) { - case "yes", "y": - confirmOK = true - case "no", "n": - os.Exit(0) - default: - continue - } - } - - return nil -} - -type UpgradeConfirm struct { - common.KubeAction -} - -func (u *UpgradeConfirm) Execute(runtime connector.Runtime) error { - pre := make([]map[string]string, len(runtime.GetAllHosts()), len(runtime.GetAllHosts())) - for i, host := range runtime.GetAllHosts() { - if v, ok := host.GetCache().Get(common.NodePreCheck); ok { - pre[i] = v.(map[string]string) - } else { - return errors.New("get node check result failed by host cache") - } - } - - results := make([]PreCheckResults, len(pre), len(pre)) - for i := range pre { - var result PreCheckResults - _ = mapstructure.Decode(pre[i], &result) - results[i] = result - } - table.OutputA(results) - fmt.Println() - - warningFlag := false - cmp, err := versionutil.MustParseSemantic(u.KubeConf.Cluster.Kubernetes.Version).Compare("v1.19.0") - if err != nil { - logger.Log.Fatalf("Failed to compare kubernetes version: %v", err) - } - if cmp == 0 || cmp == 1 { - for _, result := range results { - if len(result.Docker) != 0 { - dockerVersion, err := RefineDockerVersion(result.Docker) - if err != nil { - logger.Log.Fatalf("Failed to get docker version: %v", err) - } - cmp, err := versionutil.MustParseSemantic(dockerVersion).Compare("20.10.0") - if err != nil { - logger.Log.Fatalf("Failed to compare docker version: %v", err) - } - warningFlag = warningFlag || (cmp == -1) - } - } - if warningFlag { - fmt.Println(` -Warning: - - An old Docker version may cause the failure of upgrade. It is recommended that you upgrade Docker to 20.10+ beforehand. - - Issue: https://github.com/kubernetes/kubernetes/issues/101056`) - fmt.Print("\n") - } - } - - nodeStats, ok := u.PipelineCache.GetMustString(common.ClusterNodeStatus) - if !ok { - return errors.New("get cluster nodes status failed by pipeline cache") - } - fmt.Println("Cluster nodes status:") - fmt.Println(nodeStats + "\n") - - fmt.Println("Upgrade Confirmation:") - currentK8sVersion, ok := u.PipelineCache.GetMustString(common.K8sVersion) - if !ok { - return errors.New("get current Kubernetes version failed by pipeline cache") - } - fmt.Printf("kubernetes version: %s to %s\n", currentK8sVersion, u.KubeConf.Cluster.Kubernetes.Version) - - if u.KubeConf.Cluster.KubeSphere.Enabled { - currentKsVersion, ok := u.PipelineCache.GetMustString(common.KubeSphereVersion) - if !ok { - return errors.New("get current KubeSphere version failed by pipeline cache") - } - fmt.Printf("kubesphere version: %s to %s\n", currentKsVersion, u.KubeConf.Cluster.KubeSphere.Version) - } - fmt.Println() - - if k8sVersion, err := versionutil.ParseGeneric(u.KubeConf.Cluster.Kubernetes.Version); err == nil { - if cri, ok := u.PipelineCache.GetMustString(common.ClusterNodeCRIRuntimes); ok { - k8sV124 := versionutil.MustParseSemantic("v1.24.0") - if k8sVersion.AtLeast(k8sV124) && versionutil.MustParseSemantic(currentK8sVersion).LessThan(k8sV124) && strings.Contains(cri, "docker") { - fmt.Println("[Notice]") - fmt.Println("Pre-upgrade check failed. The container runtime of the current cluster is Docker.") - fmt.Println("Kubernetes v1.24 and later no longer support dockershim and Docker.") - fmt.Println("Make sure you have completed the migration from Docker to other container runtimes that are compatible with the Kubernetes CRI.") - fmt.Println("For more information, see:") - fmt.Println("https://kubernetes.io/docs/setup/production-environment/container-runtimes/#container-runtimes") - fmt.Println("https://kubernetes.io/blog/2022/02/17/dockershim-faq/") - fmt.Println("") - } - } - } - - reader := bufio.NewReader(os.Stdin) - confirmOK := false - for !confirmOK { - fmt.Printf("Continue upgrading cluster? [yes/no]: ") - input, err := reader.ReadString('\n') - if err != nil { - return err - } - input = strings.ToLower(strings.TrimSpace(input)) - - switch input { - case "yes", "y": - confirmOK = true - case "no", "n": - os.Exit(0) - default: - continue - } - } - return nil -} - -func RefineDockerVersion(version string) (string, error) { - var newVersionComponents []string - versionMatchRE := regexp.MustCompile(`^\s*v?([0-9]+(?:\.[0-9]+)*)(.*)*$`) - parts := versionMatchRE.FindStringSubmatch(version) - if parts == nil { - return "", fmt.Errorf("could not parse %q as version", version) - } - numbers, _ := parts[1], parts[2] - components := strings.Split(numbers, ".") - - for _, c := range components { - newVersion := strings.TrimPrefix(c, "0") - if newVersion == "" { - newVersion = "0" - } - newVersionComponents = append(newVersionComponents, newVersion) - } - return strings.Join(newVersionComponents, "."), nil -} - -type CheckFile struct { - action.BaseAction - FileName string -} - -func (c *CheckFile) Execute(runtime connector.Runtime) error { - if util.IsExist(c.FileName) { - reader := bufio.NewReader(os.Stdin) - stop := false - for { - if stop { - break - } - fmt.Printf("%s already exists. Are you sure you want to overwrite this file? [yes/no]: ", c.FileName) - input, _ := reader.ReadString('\n') - input = strings.ToLower(strings.TrimSpace(input)) - - if input != "" { - switch input { - case "yes", "y": - stop = true - case "no", "n": - os.Exit(0) - } - } - } - } - return nil -} - -type MigrateCri struct { - common.KubeAction -} - -func (d *MigrateCri) Execute(runtime connector.Runtime) error { - reader := bufio.NewReader(os.Stdin) - - confirmOK := false - for !confirmOK { - fmt.Printf("Are you sure to Migrate Cri? [yes/no]: ") - input, err := reader.ReadString('\n') - if err != nil { - return err - } - input = strings.ToLower(strings.TrimSpace(input)) - - switch strings.ToLower(input) { - case "yes", "y": - confirmOK = true - case "no", "n": - os.Exit(0) - default: - continue - } - } - - return nil -} diff --git a/cmd/kk/pkg/bootstrap/confirm/tasks_test.go b/cmd/kk/pkg/bootstrap/confirm/tasks_test.go deleted file mode 100644 index 63c10aa9e..000000000 --- a/cmd/kk/pkg/bootstrap/confirm/tasks_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package confirm - -import "testing" - -func TestRefineDockerVersion(t *testing.T) { - cases := map[string]string{ - "1.2.3": "1.2.3", - "23.0.1": "23.0.1", - " 1.2.3 ": "1.2.3", - " 23.0.1 ": "23.0.1", - "abc": "err", - } - - for k, v := range cases { - result, err := RefineDockerVersion(k) - if err != nil { - if v != "err" { - t.Errorf("case \"%s\", result \"%s\" expected, but \"%s\" get", k, v, "err") - } - } else { - if result != v { - t.Errorf("case \"%s\", result \"%s\" expected, but \"%s\" get", k, v, result) - } - } - } -} diff --git a/cmd/kk/pkg/bootstrap/customscripts/module.go b/cmd/kk/pkg/bootstrap/customscripts/module.go deleted file mode 100644 index 1c98a178d..000000000 --- a/cmd/kk/pkg/bootstrap/customscripts/module.go +++ /dev/null @@ -1,52 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package customscripts - -import ( - "fmt" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" -) - -type CustomScriptsModule struct { - module.BaseTaskModule - Phase string - Scripts []kubekeyapiv1alpha2.CustomScripts -} - -func (m *CustomScriptsModule) Init() { - m.Name = fmt.Sprintf("CustomScriptsModule Phase:%s", m.Phase) - m.Desc = "Exec custom shell scripts for each nodes." - - for idx, script := range m.Scripts { - - taskName := fmt.Sprintf("Phase:%s(%d/%d) script:%s", m.Phase, idx, len(m.Scripts), script.Name) - taskDir := fmt.Sprintf("%s-%d-script", m.Phase, idx) - task := &task.RemoteTask{ - Name: taskName, - Desc: taskName, - Hosts: m.Runtime.GetAllHosts(), - Action: &CustomScriptTask{taskDir: taskDir, script: script}, - Parallel: true, - Retry: 1, - } - - m.Tasks = append(m.Tasks, task) - } -} diff --git a/cmd/kk/pkg/bootstrap/customscripts/tasks.go b/cmd/kk/pkg/bootstrap/customscripts/tasks.go deleted file mode 100644 index 439e4dc0d..000000000 --- a/cmd/kk/pkg/bootstrap/customscripts/tasks.go +++ /dev/null @@ -1,117 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package customscripts - -import ( - "fmt" - "os" - "path/filepath" - "strings" - "time" - - "github.com/pkg/errors" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -type CustomScriptTask struct { - action.BaseAction - taskDir string - script kubekeyapiv1alpha2.CustomScripts -} - -func (t *CustomScriptTask) Execute(runtime connector.Runtime) error { - - if len(t.script.Bash) <= 0 { - return errors.Errorf("custom script %s Bash is empty", t.script.Name) - } - - remoteTaskHome := common.TmpDir + t.taskDir - - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("mkdir -p %s", remoteTaskHome), false); err != nil { - return errors.Wrapf(err, "create remoteTaskHome: %s err:%s", remoteTaskHome, err) - } - - // dilver the dependency materials to the remotehost - for idx, localPath := range t.script.Materials { - - if !util.IsExist(localPath) { - return errors.Errorf("Not found Path: %s", localPath) - } - - targetPath := filepath.Join(remoteTaskHome, filepath.Base(localPath)) - - // first clean the target to makesure target path always is the lastest. - cleanCmd := fmt.Sprintf("rm -fr %s", targetPath) - if _, err := runtime.GetRunner().SudoCmd(cleanCmd, false); err != nil { - return errors.Wrapf(err, "Can not remove target found Path: %s", targetPath) - } - - start := time.Now() - err := runtime.GetRunner().SudoScp(localPath, targetPath) - if err != nil { - return errors.Wrapf(err, "Can not Scp -fr %s root@%s:%s", localPath, runtime.RemoteHost().GetAddress(), targetPath) - } - - fmt.Printf("Copy %d/%d materials: Scp -fr %s root@%s:%s done, take %s\n", - idx, len(t.script.Materials), localPath, runtime.RemoteHost().GetAddress(), targetPath, time.Since(start)) - } - - // wrap use bash file if shell has many lines. - RunBash := t.script.Bash - if strings.Index(RunBash, "\n") > 0 { - tmpFile, err := os.CreateTemp(os.TempDir(), t.taskDir) - if err != nil { - return errors.Wrapf(err, "create tmp Bash: %s/%s in local node, err:%s", os.TempDir(), t.taskDir, err) - } - defer os.Remove(tmpFile.Name()) - - if _, err := tmpFile.WriteString(RunBash); err != nil { - return errors.Wrapf(err, "write to tmp:%s in local node, err:%s", tmpFile.Name(), err) - } - - targetPath := filepath.Join(remoteTaskHome, "task.sh") - if err := runtime.GetRunner().SudoScp(tmpFile.Name(), targetPath); err != nil { - return errors.Wrapf(err, "Can not Scp -fr %s root@%s:%s", tmpFile.Name(), runtime.RemoteHost().GetAddress(), targetPath) - } - - RunBash = "/bin/bash " + targetPath - } - - start := time.Now() - out, err := runtime.GetRunner().SudoCmd(RunBash, false) - if err != nil { - return errors.Errorf("Exec Bash: %s err:%s", RunBash, err) - } - - if !runtime.GetRunner().Debug { - fmt.Printf("Exec Bash:%s done, take %s", RunBash, time.Since(start)) - cleanCmd := fmt.Sprintf("rm -fr %s", remoteTaskHome) - if _, err := runtime.GetRunner().SudoCmd(cleanCmd, false); err != nil { - return errors.Wrapf(err, "Exec cmd:%s err:%s", cleanCmd, err) - } - } else { - // keep the Materials for debug - fmt.Printf("Exec Bash:%s done, take %s, output:\n%s", RunBash, time.Since(start), out) - } - - return nil -} diff --git a/cmd/kk/pkg/bootstrap/os/common.go b/cmd/kk/pkg/bootstrap/os/common.go deleted file mode 100644 index 59ce5c29b..000000000 --- a/cmd/kk/pkg/bootstrap/os/common.go +++ /dev/null @@ -1,21 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package os - -const ( - Release = "release" -) diff --git a/cmd/kk/pkg/bootstrap/os/module.go b/cmd/kk/pkg/bootstrap/os/module.go deleted file mode 100644 index ef34ae4ef..000000000 --- a/cmd/kk/pkg/bootstrap/os/module.go +++ /dev/null @@ -1,352 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package os - -import ( - "path/filepath" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -type ConfigureOSModule struct { - common.KubeModule - Skip bool -} - -func (c *ConfigureOSModule) IsSkip() bool { - return c.Skip -} - -func (c *ConfigureOSModule) Init() { - c.Name = "ConfigureOSModule" - c.Desc = "Init os dependencies" - - getOSData := &task.RemoteTask{ - Name: "GetOSData", - Desc: "Get OS release", - Hosts: c.Runtime.GetAllHosts(), - Action: new(GetOSData), - Parallel: true, - } - - initOS := &task.RemoteTask{ - Name: "InitOS", - Desc: "Prepare to init OS", - Hosts: c.Runtime.GetAllHosts(), - Action: new(NodeConfigureOS), - Parallel: true, - } - - GenerateScript := &task.RemoteTask{ - Name: "GenerateScript", - Desc: "Generate init os script", - Hosts: c.Runtime.GetAllHosts(), - Action: &action.Template{ - Template: templates.InitOsScriptTmpl, - Dst: filepath.Join(common.KubeScriptDir, "initOS.sh"), - Data: util.Data{ - "Hosts": templates.GenerateHosts(c.Runtime, c.KubeConf), - }, - }, - Parallel: true, - } - - ExecScript := &task.RemoteTask{ - Name: "ExecScript", - Desc: "Exec init os script", - Hosts: c.Runtime.GetAllHosts(), - Action: new(NodeExecScript), - Parallel: true, - } - - ConfigureNtpServer := &task.RemoteTask{ - Name: "ConfigureNtpServer", - Desc: "configure the ntp server for each node", - Hosts: c.Runtime.GetAllHosts(), - Prepare: new(NodeConfigureNtpCheck), - Action: new(NodeConfigureNtpServer), - Parallel: true, - } - - c.Tasks = []task.Interface{ - getOSData, - initOS, - GenerateScript, - ExecScript, - ConfigureNtpServer, - } -} - -type ClearNodeOSModule struct { - common.KubeModule -} - -func (c *ClearNodeOSModule) Init() { - c.Name = "ClearNodeOSModule" - - stopKubelet := &task.RemoteTask{ - Name: "StopKubelet", - Desc: "Stop Kubelet", - Hosts: c.Runtime.GetHostsByRole(common.Worker), - Prepare: new(DeleteNode), - Action: new(StopKubelet), - Parallel: true, - } - - resetNetworkConfig := &task.RemoteTask{ - Name: "ResetNetworkConfig", - Desc: "Reset os network config", - Hosts: c.Runtime.GetHostsByRole(common.Worker), - Prepare: new(DeleteNode), - Action: new(ResetNetworkConfig), - Parallel: true, - } - - removeFiles := &task.RemoteTask{ - Name: "RemoveFiles", - Desc: "Remove node files", - Hosts: c.Runtime.GetHostsByRole(common.Worker), - Prepare: new(DeleteNode), - Action: new(RemoveNodeFiles), - Parallel: true, - } - - daemonReload := &task.RemoteTask{ - Name: "DaemonReload", - Desc: "Systemd daemon reload", - Hosts: c.Runtime.GetHostsByRole(common.Worker), - Prepare: new(DeleteNode), - Action: new(DaemonReload), - Parallel: true, - } - - c.Tasks = []task.Interface{ - stopKubelet, - resetNetworkConfig, - removeFiles, - daemonReload, - } -} - -type ClearOSEnvironmentModule struct { - common.KubeModule -} - -func (c *ClearOSEnvironmentModule) Init() { - c.Name = "ClearOSModule" - - resetNetworkConfig := &task.RemoteTask{ - Name: "ResetNetworkConfig", - Desc: "Reset os network config", - Hosts: c.Runtime.GetHostsByRole(common.K8s), - Action: new(ResetNetworkConfig), - Parallel: true, - } - - uninstallETCD := &task.RemoteTask{ - Name: "UninstallETCD", - Desc: "Uninstall etcd", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Prepare: &prepare.PrepareCollection{ - new(EtcdTypeIsKubeKey), - }, - Action: new(UninstallETCD), - Parallel: true, - } - - removeFiles := &task.RemoteTask{ - Name: "RemoveFiles", - Desc: "Remove cluster files", - Hosts: c.Runtime.GetHostsByRole(common.K8s), - Action: new(RemoveFiles), - Parallel: true, - } - - daemonReload := &task.RemoteTask{ - Name: "DaemonReload", - Desc: "Systemd daemon reload", - Hosts: c.Runtime.GetHostsByRole(common.K8s), - Action: new(DaemonReload), - Parallel: true, - } - - c.Tasks = []task.Interface{ - resetNetworkConfig, - uninstallETCD, - removeFiles, - daemonReload, - } -} - -type RepositoryOnlineModule struct { - common.KubeModule - Skip bool -} - -func (r *RepositoryOnlineModule) IsSkip() bool { - return r.Skip -} - -func (r *RepositoryOnlineModule) Init() { - r.Name = "RepositoryOnlineModule" - - getOSData := &task.RemoteTask{ - Name: "GetOSData", - Desc: "Get OS release", - Hosts: r.Runtime.GetAllHosts(), - Action: new(GetOSData), - Parallel: true, - } - - newRepo := &task.RemoteTask{ - Name: "NewRepoClient", - Desc: "New repository client", - Hosts: r.Runtime.GetAllHosts(), - Action: new(NewRepoClient), - Parallel: true, - Retry: 1, - } - - install := &task.RemoteTask{ - Name: "InstallPackage", - Desc: "Install packages", - Hosts: r.Runtime.GetAllHosts(), - Action: new(InstallPackage), - Parallel: true, - Retry: 1, - } - - r.Tasks = []task.Interface{ - getOSData, - newRepo, - install, - } -} - -type RepositoryModule struct { - common.KubeModule - Skip bool -} - -func (r *RepositoryModule) IsSkip() bool { - return r.Skip -} - -func (r *RepositoryModule) Init() { - r.Name = "RepositoryModule" - r.Desc = "Install local repository" - - getOSData := &task.RemoteTask{ - Name: "GetOSData", - Desc: "Get OS release", - Hosts: r.Runtime.GetAllHosts(), - Action: new(GetOSData), - Parallel: true, - } - - sync := &task.RemoteTask{ - Name: "SyncRepositoryISOFile", - Desc: "Sync repository iso file to all nodes", - Hosts: r.Runtime.GetAllHosts(), - Action: new(SyncRepositoryFile), - Parallel: true, - Retry: 2, - } - - mount := &task.RemoteTask{ - Name: "MountISO", - Desc: "Mount iso file", - Hosts: r.Runtime.GetAllHosts(), - Action: new(MountISO), - Parallel: true, - Retry: 1, - } - - newRepo := &task.RemoteTask{ - Name: "NewRepoClient", - Desc: "New repository client", - Hosts: r.Runtime.GetAllHosts(), - Action: new(NewRepoClient), - Parallel: true, - Retry: 1, - Rollback: new(RollbackUmount), - } - - backup := &task.RemoteTask{ - Name: "BackupOriginalRepository", - Desc: "Backup original repository", - Hosts: r.Runtime.GetAllHosts(), - Action: new(BackupOriginalRepository), - Parallel: true, - Retry: 1, - Rollback: new(RecoverBackupSuccessNode), - } - - add := &task.RemoteTask{ - Name: "AddLocalRepository", - Desc: "Add local repository", - Hosts: r.Runtime.GetAllHosts(), - Action: new(AddLocalRepository), - Parallel: true, - Retry: 1, - Rollback: new(RecoverRepository), - } - - install := &task.RemoteTask{ - Name: "InstallPackage", - Desc: "Install packages", - Hosts: r.Runtime.GetAllHosts(), - Action: new(InstallPackage), - Parallel: true, - Retry: 1, - Rollback: new(RecoverRepository), - } - - reset := &task.RemoteTask{ - Name: "ResetRepository", - Desc: "Reset repository to the original repository", - Hosts: r.Runtime.GetAllHosts(), - Action: new(ResetRepository), - Parallel: true, - Retry: 1, - } - - umount := &task.RemoteTask{ - Name: "UmountISO", - Desc: "Umount ISO file", - Hosts: r.Runtime.GetAllHosts(), - Action: new(UmountISO), - Parallel: true, - } - - r.Tasks = []task.Interface{ - getOSData, - sync, - mount, - newRepo, - backup, - add, - install, - reset, - umount, - } -} diff --git a/cmd/kk/pkg/bootstrap/os/prepares.go b/cmd/kk/pkg/bootstrap/os/prepares.go deleted file mode 100644 index 679004131..000000000 --- a/cmd/kk/pkg/bootstrap/os/prepares.go +++ /dev/null @@ -1,66 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package os - -import ( - kubekeyv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type NodeConfigureNtpCheck struct { - common.KubePrepare -} - -func (n *NodeConfigureNtpCheck) PreCheck(_ connector.Runtime) (bool, error) { - // skip when both NtpServers and Timezone was not set in cluster config - if len(n.KubeConf.Cluster.System.NtpServers) == 0 && len(n.KubeConf.Cluster.System.Timezone) == 0 { - return false, nil - } - - return true, nil -} - -type EtcdTypeIsKubeKey struct { - common.KubePrepare -} - -func (e *EtcdTypeIsKubeKey) PreCheck(_ connector.Runtime) (bool, error) { - if len(e.KubeConf.Cluster.Etcd.Type) == 0 || e.KubeConf.Cluster.Etcd.Type == kubekeyv1alpha2.KubeKey { - return true, nil - } - - return false, nil -} - -type DeleteNode struct { - common.KubePrepare -} - -func (d *DeleteNode) PreCheck(runtime connector.Runtime) (bool, error) { - nodeName, ok := d.PipelineCache.Get("dstNode") - if !ok { - return true, nil - } - - host := runtime.RemoteHost() - if host.GetName() == nodeName { - return true, nil - } - - return false, nil -} diff --git a/cmd/kk/pkg/bootstrap/os/repository/repository.go b/cmd/kk/pkg/bootstrap/os/repository/repository.go deleted file mode 100644 index 89ae3de58..000000000 --- a/cmd/kk/pkg/bootstrap/os/repository/repository.go +++ /dev/null @@ -1,44 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package repository - -import ( - "fmt" - "strings" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type Interface interface { - Backup(runtime connector.Runtime) error - IsAlreadyBackUp() bool - Add(runtime connector.Runtime, path string) error - Update(runtime connector.Runtime) error - Install(runtime connector.Runtime, pkg ...string) error - Reset(runtime connector.Runtime) error -} - -func New(os string) (Interface, error) { - switch strings.ToLower(os) { - case "ubuntu", "debian": - return NewDeb(), nil - case "centos", "rhel": - return NewRPM(), nil - default: - return nil, fmt.Errorf("unsupported operation system %s", os) - } -} diff --git a/cmd/kk/pkg/bootstrap/os/repository/repository_deb.go b/cmd/kk/pkg/bootstrap/os/repository/repository_deb.go deleted file mode 100644 index f535fe786..000000000 --- a/cmd/kk/pkg/bootstrap/os/repository/repository_deb.go +++ /dev/null @@ -1,106 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package repository - -import ( - "fmt" - "strings" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type Debian struct { - backup bool -} - -func NewDeb() Interface { - return &Debian{} -} - -func (d *Debian) Backup(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("mv /etc/apt/sources.list /etc/apt/sources.list.kubekey.bak", false); err != nil { - return err - } - - if _, err := runtime.GetRunner().SudoCmd("mv /etc/apt/sources.list.d /etc/apt/sources.list.d.kubekey.bak", false); err != nil { - return err - } - - if _, err := runtime.GetRunner().SudoCmd("mkdir -p /etc/apt/sources.list.d", false); err != nil { - return err - } - d.backup = true - return nil -} - -func (d *Debian) IsAlreadyBackUp() bool { - return d.backup -} - -func (d *Debian) Add(runtime connector.Runtime, path string) error { - if !d.IsAlreadyBackUp() { - return fmt.Errorf("linux repository must be backuped before") - } - - if _, err := runtime.GetRunner().SudoCmd("rm -rf /etc/apt/sources.list.d/*", false); err != nil { - return err - } - - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("echo 'deb [trusted=yes] file://%s /' > /etc/apt/sources.list.d/kubekey.list", path), - true); err != nil { - return err - } - return nil -} - -func (d *Debian) Update(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().Cmd("sudo apt-get update", true); err != nil { - return err - } - return nil -} - -func (d *Debian) Install(runtime connector.Runtime, pkg ...string) error { - defaultPkg := []string{"socat", "conntrack", "ipset", "ebtables", "chrony", "ipvsadm"} - if len(pkg) == 0 { - pkg = defaultPkg - } else { - pkg = append(pkg, defaultPkg...) - } - - str := strings.Join(pkg, " ") - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("apt install -y %s", str), true); err != nil { - return err - } - return nil -} - -func (d *Debian) Reset(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("rm -rf /etc/apt/sources.list.d", false); err != nil { - return err - } - - if _, err := runtime.GetRunner().SudoCmd("mv /etc/apt/sources.list.kubekey.bak /etc/apt/sources.list", false); err != nil { - return err - } - - if _, err := runtime.GetRunner().SudoCmd("mv /etc/apt/sources.list.d.kubekey.bak /etc/apt/sources.list.d", false); err != nil { - return err - } - - return nil -} diff --git a/cmd/kk/pkg/bootstrap/os/repository/repository_rpm.go b/cmd/kk/pkg/bootstrap/os/repository/repository_rpm.go deleted file mode 100644 index 166062564..000000000 --- a/cmd/kk/pkg/bootstrap/os/repository/repository_rpm.go +++ /dev/null @@ -1,110 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package repository - -import ( - "fmt" - "strings" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type RedhatPackageManager struct { - backup bool -} - -func NewRPM() Interface { - return &RedhatPackageManager{} -} - -func (r *RedhatPackageManager) Backup(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("mv /etc/yum.repos.d /etc/yum.repos.d.kubekey.bak", false); err != nil { - return err - } - - if _, err := runtime.GetRunner().SudoCmd("mkdir -p /etc/yum.repos.d", false); err != nil { - return err - } - r.backup = true - return nil -} - -func (r *RedhatPackageManager) IsAlreadyBackUp() bool { - return r.backup -} - -func (r *RedhatPackageManager) Add(runtime connector.Runtime, path string) error { - if !r.IsAlreadyBackUp() { - return fmt.Errorf("linux repository must be backuped before") - } - - if _, err := runtime.GetRunner().SudoCmd("rm -rf /etc/yum.repos.d/*", false); err != nil { - return err - } - - content := fmt.Sprintf(`cat << EOF > /etc/yum.repos.d/CentOS-local.repo -[base-local] -name=rpms-local - -baseurl=file://%s - -enabled=1 - -gpgcheck=0 - -EOF -`, path) - if _, err := runtime.GetRunner().SudoCmd(content, false); err != nil { - return err - } - - return nil -} - -func (r *RedhatPackageManager) Update(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("yum clean all && yum makecache", true); err != nil { - return err - } - return nil -} - -func (r *RedhatPackageManager) Install(runtime connector.Runtime, pkg ...string) error { - defaultPkg := []string{"openssl", "socat", "conntrack", "ipset", "ebtables", "chrony", "ipvsadm"} - if len(pkg) == 0 { - pkg = defaultPkg - } else { - pkg = append(pkg, defaultPkg...) - } - - str := strings.Join(pkg, " ") - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("yum install -y %s", str), true); err != nil { - return err - } - return nil -} - -func (r *RedhatPackageManager) Reset(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("rm -rf /etc/yum.repos.d", false); err != nil { - return err - } - - if _, err := runtime.GetRunner().SudoCmd("mv /etc/yum.repos.d.kubekey.bak /etc/yum.repos.d", false); err != nil { - return err - } - - return nil -} diff --git a/cmd/kk/pkg/bootstrap/os/rollback.go b/cmd/kk/pkg/bootstrap/os/rollback.go deleted file mode 100644 index d6eb6655e..000000000 --- a/cmd/kk/pkg/bootstrap/os/rollback.go +++ /dev/null @@ -1,90 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package os - -import ( - "fmt" - "path/filepath" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os/repository" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/ending" -) - -type RollbackUmount struct { - common.KubeRollback -} - -func (r *RollbackUmount) Execute(runtime connector.Runtime, result *ending.ActionResult) error { - mountPath := filepath.Join(common.TmpDir, "iso") - umountCmd := fmt.Sprintf("umount %s", mountPath) - if _, err := runtime.GetRunner().SudoCmd(umountCmd, false); err != nil { - return errors.Wrapf(errors.WithStack(err), "umount %s failed", mountPath) - } - return nil -} - -type RecoverBackupSuccessNode struct { - common.KubeRollback -} - -func (r *RecoverBackupSuccessNode) Execute(runtime connector.Runtime, result *ending.ActionResult) error { - if result.Status == ending.SUCCESS { - host := runtime.RemoteHost() - repo, ok := host.GetCache().Get("repo") - if !ok { - return errors.New("get repo failed by host cache") - } - - re := repo.(repository.Interface) - if err := re.Reset(runtime); err != nil { - return errors.Wrapf(errors.WithStack(err), "reset repository failed") - } - } - - mountPath := filepath.Join(common.TmpDir, "iso") - umountCmd := fmt.Sprintf("umount %s", mountPath) - if _, err := runtime.GetRunner().SudoCmd(umountCmd, false); err != nil { - return errors.Wrapf(errors.WithStack(err), "umount %s failed", mountPath) - } - return nil -} - -type RecoverRepository struct { - common.KubeRollback -} - -func (r *RecoverRepository) Execute(runtime connector.Runtime, result *ending.ActionResult) error { - host := runtime.RemoteHost() - repo, ok := host.GetCache().Get("repo") - if !ok { - return errors.New("get repo failed by host cache") - } - - re := repo.(repository.Interface) - _ = re.Reset(runtime) - - mountPath := filepath.Join(common.TmpDir, "iso") - umountCmd := fmt.Sprintf("umount %s", mountPath) - if _, err := runtime.GetRunner().SudoCmd(umountCmd, false); err != nil { - return errors.Wrapf(errors.WithStack(err), "umount %s failed", mountPath) - } - return nil -} diff --git a/cmd/kk/pkg/bootstrap/os/tasks.go b/cmd/kk/pkg/bootstrap/os/tasks.go deleted file mode 100644 index 1f3467610..000000000 --- a/cmd/kk/pkg/bootstrap/os/tasks.go +++ /dev/null @@ -1,559 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package os - -import ( - "fmt" - "path/filepath" - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os/repository" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" - "github.com/kubesphere/kubekey/v3/util/osrelease" -) - -type NodeConfigureOS struct { - common.KubeAction -} - -func (n *NodeConfigureOS) Execute(runtime connector.Runtime) error { - - host := runtime.RemoteHost() - if err := addUsers(runtime, host); err != nil { - return errors.Wrap(errors.WithStack(err), "Failed to add users") - } - - if err := createDirectories(runtime, host); err != nil { - return err - } - - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - _, err1 := runtime.GetRunner().SudoCmd(fmt.Sprintf("hostnamectl set-hostname %s && sed -i '/^127.0.1.1/s/.*/127.0.1.1 %s/g' /etc/hosts", host.GetName(), host.GetName()), false) - if err1 != nil { - return errors.Wrap(errors.WithStack(err1), "Failed to override hostname") - } - - return nil -} - -func addUsers(runtime connector.Runtime, node connector.Host) error { - if _, err := runtime.GetRunner().SudoCmd("useradd -M -c 'Kubernetes user' -s /sbin/nologin -r kube || :", false); err != nil { - return err - } - - if node.IsRole(common.ETCD) { - if _, err := runtime.GetRunner().SudoCmd("useradd -M -c 'Etcd user' -s /sbin/nologin -r etcd || :", false); err != nil { - return err - } - } - - return nil -} - -func createDirectories(runtime connector.Runtime, node connector.Host) error { - dirs := []string{ - common.BinDir, - common.KubeConfigDir, - common.KubeCertDir, - common.KubeManifestDir, - common.KubeScriptDir, - common.KubeletFlexvolumesPluginsDir, - } - - for _, dir := range dirs { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("mkdir -p %s", dir), false); err != nil { - return err - } - if dir == common.KubeletFlexvolumesPluginsDir { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chown kube -R %s", "/usr/libexec/kubernetes"), false); err != nil { - return err - } - } else { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chown kube -R %s", dir), false); err != nil { - return err - } - } - } - - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("mkdir -p %s && chown kube -R %s", "/etc/cni/net.d", "/etc/cni"), false); err != nil { - return err - } - - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("mkdir -p %s && chown kube -R %s", "/opt/cni/bin", "/opt/cni"), false); err != nil { - return err - } - - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("mkdir -p %s && chown kube -R %s", "/var/lib/calico", "/var/lib/calico"), false); err != nil { - return err - } - - if node.IsRole(common.ETCD) { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("mkdir -p %s && chown etcd -R %s", "/var/lib/etcd", "/var/lib/etcd"), false); err != nil { - return err - } - } - - return nil -} - -type NodeExecScript struct { - common.KubeAction -} - -func (n *NodeExecScript) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chmod +x %s/initOS.sh", common.KubeScriptDir), false); err != nil { - return errors.Wrap(errors.WithStack(err), "Failed to chmod +x init os script") - } - - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("%s/initOS.sh", common.KubeScriptDir), true); err != nil { - return errors.Wrap(errors.WithStack(err), "Failed to configure operating system") - } - return nil -} - -var ( - etcdFiles = []string{ - "/usr/local/bin/etcd", - "/etc/ssl/etcd", - "/var/lib/etcd/*", - "/etc/etcd.env", - } - clusterFiles = []string{ - "/etc/kubernetes", - "/etc/systemd/system/etcd.service", - "/etc/systemd/system/backup-etcd.service", - "/etc/systemd/system/backup-etcd.timer", - "/var/log/calico", - "/etc/cni", - "/var/log/pods/", - "/var/lib/cni", - "/var/lib/calico", - "/var/lib/kubelet/*", - "/run/calico", - "/run/flannel", - "/etc/flannel", - "/var/openebs", - "/etc/systemd/system/kubelet.service", - "/etc/systemd/system/kubelet.service.d", - "/usr/local/bin/kubelet", - "/usr/local/bin/kubeadm", - "/usr/bin/kubelet", - "/var/lib/rook", - "/tmp/kubekey", - "/etc/kubekey", - } - - networkResetCmds = []string{ - "iptables -F", - "iptables -X", - "iptables -F -t nat", - "iptables -X -t nat", - "ipvsadm -C", - "ip link del kube-ipvs0", - "ip link del nodelocaldns", - "ip link del cni0", - "ip link del flannel.1", - "ip link del flannel-v6.1", - "ip link del flannel-wg", - "ip link del flannel-wg-v6", - "ip link del cilium_host", - "ip link del cilium_vxlan", - "ip link del vxlan.calico", - "ip link del vxlan-v6.calico", - "ip -br link show | grep 'cali[a-f0-9]*' | awk -F '@' '{print $1}' | xargs -r -t -n 1 ip link del", - "ip netns show 2>/dev/null | grep cni- | xargs -r -t -n 1 ip netns del", - } -) - -type ResetNetworkConfig struct { - common.KubeAction -} - -func (r *ResetNetworkConfig) Execute(runtime connector.Runtime) error { - for _, cmd := range networkResetCmds { - _, _ = runtime.GetRunner().SudoCmd(cmd, true) - } - return nil -} - -type StopKubelet struct { - common.KubeAction -} - -func (s *StopKubelet) Execute(runtime connector.Runtime) error { - _, _ = runtime.GetRunner().SudoCmd("systemctl disable kubelet && systemctl stop kubelet && exit 0", false) - return nil -} - -type UninstallETCD struct { - common.KubeAction -} - -func (s *UninstallETCD) Execute(runtime connector.Runtime) error { - _, _ = runtime.GetRunner().SudoCmd("systemctl stop etcd && exit 0", false) - for _, file := range etcdFiles { - _, _ = runtime.GetRunner().SudoCmd(fmt.Sprintf("rm -rf %s", file), true) - } - return nil -} - -type RemoveNodeFiles struct { - common.KubeAction -} - -func (r *RemoveNodeFiles) Execute(runtime connector.Runtime) error { - nodeFiles := []string{ - "/etc/kubernetes", - "/etc/systemd/system/etcd.service", - "/var/log/calico", - "/etc/cni", - "/var/log/pods/", - "/var/lib/cni", - "/var/lib/calico", - "/var/lib/kubelet/*", - "/run/calico", - "/run/flannel", - "/etc/flannel", - "/etc/systemd/system/kubelet.service", - "/etc/systemd/system/kubelet.service.d", - "/usr/local/bin/kubelet", - "/usr/local/bin/kubeadm", - "/usr/bin/kubelet", - "/tmp/kubekey", - "/etc/kubekey", - } - - for _, file := range nodeFiles { - _, _ = runtime.GetRunner().SudoCmd(fmt.Sprintf("rm -rf %s", file), true) - } - return nil -} - -type RemoveFiles struct { - common.KubeAction -} - -func (r *RemoveFiles) Execute(runtime connector.Runtime) error { - for _, file := range clusterFiles { - _, _ = runtime.GetRunner().SudoCmd(fmt.Sprintf("rm -rf %s", file), true) - } - return nil -} - -type DaemonReload struct { - common.KubeAction -} - -func (d *DaemonReload) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && exit 0", false); err != nil { - return errors.Wrap(errors.WithStack(err), "systemctl daemon-reload failed") - } - - // try to restart the cotainerd after /etc/cni has been removed - _, _ = runtime.GetRunner().SudoCmd("systemctl restart containerd", false) - return nil -} - -type GetOSData struct { - common.KubeAction -} - -func (g *GetOSData) Execute(runtime connector.Runtime) error { - osReleaseStr, err := runtime.GetRunner().SudoCmd("cat /etc/os-release", false) - if err != nil { - return err - } - osrData := osrelease.Parse(strings.Replace(osReleaseStr, "\r\n", "\n", -1)) - - host := runtime.RemoteHost() - // type: *osrelease.data - host.GetCache().Set(Release, osrData) - return nil -} - -type SyncRepositoryFile struct { - common.KubeAction -} - -func (s *SyncRepositoryFile) Execute(runtime connector.Runtime) error { - if err := utils.ResetTmpDir(runtime); err != nil { - return errors.Wrap(err, "reset tmp dir failed") - } - - host := runtime.RemoteHost() - release, ok := host.GetCache().Get(Release) - if !ok { - return errors.New("get os release failed by root cache") - } - r := release.(*osrelease.Data) - - fileName := fmt.Sprintf("%s-%s-%s.iso", r.ID, r.VersionID, host.GetArch()) - src := filepath.Join(runtime.GetWorkDir(), "repository", host.GetArch(), r.ID, r.VersionID, fileName) - dst := filepath.Join(common.TmpDir, fileName) - if err := runtime.GetRunner().Scp(src, dst); err != nil { - return errors.Wrapf(errors.WithStack(err), "scp %s to %s failed", src, dst) - } - - host.GetCache().Set("iso", fileName) - return nil -} - -type MountISO struct { - common.KubeAction -} - -func (m *MountISO) Execute(runtime connector.Runtime) error { - mountPath := filepath.Join(common.TmpDir, "iso") - if err := runtime.GetRunner().MkDir(mountPath); err != nil { - return errors.Wrapf(errors.WithStack(err), "create mount dir failed") - } - - host := runtime.RemoteHost() - isoFile, _ := host.GetCache().GetMustString("iso") - path := filepath.Join(common.TmpDir, isoFile) - mountCmd := fmt.Sprintf("sudo mount -t iso9660 -o loop %s %s", path, mountPath) - if _, err := runtime.GetRunner().Cmd(mountCmd, false); err != nil { - return errors.Wrapf(errors.WithStack(err), "mount %s at %s failed", path, mountPath) - } - return nil -} - -type NewRepoClient struct { - common.KubeAction -} - -func (n *NewRepoClient) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - release, ok := host.GetCache().Get(Release) - if !ok { - return errors.New("get os release failed by host cache") - } - r := release.(*osrelease.Data) - - repo, err := repository.New(r.ID) - if err != nil { - checkDeb, debErr := runtime.GetRunner().SudoCmd("which apt", false) - if debErr == nil && strings.Contains(checkDeb, "bin") { - repo = repository.NewDeb() - } - checkRPM, rpmErr := runtime.GetRunner().SudoCmd("which yum", false) - if rpmErr == nil && strings.Contains(checkRPM, "bin") { - repo = repository.NewRPM() - } - - if debErr != nil && rpmErr != nil { - return errors.Wrap(errors.WithStack(err), "new repository manager failed") - } else if debErr == nil && rpmErr == nil { - return errors.New("can't detect the main package repository, only one of apt or yum is supported") - } - } - - host.GetCache().Set("repo", repo) - return nil -} - -type BackupOriginalRepository struct { - common.KubeAction -} - -func (b *BackupOriginalRepository) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - r, ok := host.GetCache().Get("repo") - if !ok { - return errors.New("get repo failed by host cache") - } - repo := r.(repository.Interface) - - if err := repo.Backup(runtime); err != nil { - return errors.Wrap(errors.WithStack(err), "backup repository failed") - } - - return nil -} - -type AddLocalRepository struct { - common.KubeAction -} - -func (a *AddLocalRepository) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - r, ok := host.GetCache().Get("repo") - if !ok { - return errors.New("get repo failed by host cache") - } - repo := r.(repository.Interface) - - if installErr := repo.Add(runtime, filepath.Join(common.TmpDir, "iso")); installErr != nil { - return errors.Wrap(errors.WithStack(installErr), "add local repository failed") - } - if installErr := repo.Update(runtime); installErr != nil { - return errors.Wrap(errors.WithStack(installErr), "update local repository failed") - } - - return nil -} - -type InstallPackage struct { - common.KubeAction -} - -func (i *InstallPackage) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - repo, ok := host.GetCache().Get("repo") - if !ok { - return errors.New("get repo failed by host cache") - } - r := repo.(repository.Interface) - - var pkg []string - if _, ok := r.(*repository.Debian); ok { - pkg = i.KubeConf.Cluster.System.Debs - } else if _, ok := r.(*repository.RedhatPackageManager); ok { - pkg = i.KubeConf.Cluster.System.Rpms - } - - if installErr := r.Update(runtime); installErr != nil { - return errors.Wrap(errors.WithStack(installErr), "update repository failed") - } - - if installErr := r.Install(runtime, pkg...); installErr != nil { - return errors.Wrap(errors.WithStack(installErr), "install repository package failed") - } - return nil -} - -type ResetRepository struct { - common.KubeAction -} - -func (r *ResetRepository) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - repo, ok := host.GetCache().Get("repo") - if !ok { - return errors.New("get repo failed by host cache") - } - re := repo.(repository.Interface) - - var resetErr error - defer func() { - if resetErr != nil { - mountPath := filepath.Join(common.TmpDir, "iso") - umountCmd := fmt.Sprintf("umount %s", mountPath) - _, _ = runtime.GetRunner().SudoCmd(umountCmd, false) - } - }() - - if resetErr = re.Reset(runtime); resetErr != nil { - return errors.Wrap(errors.WithStack(resetErr), "reset repository failed") - } - - return nil -} - -type UmountISO struct { - common.KubeAction -} - -func (u *UmountISO) Execute(runtime connector.Runtime) error { - mountPath := filepath.Join(common.TmpDir, "iso") - umountCmd := fmt.Sprintf("umount %s", mountPath) - if _, err := runtime.GetRunner().SudoCmd(umountCmd, false); err != nil { - return errors.Wrapf(errors.WithStack(err), "umount %s failed", mountPath) - } - return nil -} - -type NodeConfigureNtpServer struct { - common.KubeAction -} - -func (n *NodeConfigureNtpServer) Execute(runtime connector.Runtime) error { - - currentHost := runtime.RemoteHost() - release, ok := currentHost.GetCache().Get(Release) - if !ok { - return errors.New("get os release failed by host cache") - } - r := release.(*osrelease.Data) - - chronyConfigFile := "/etc/chrony.conf" - chronyService := "chronyd.service" - if r.ID == "ubuntu" || r.ID == "debian" { - chronyConfigFile = "/etc/chrony/chrony.conf" - chronyService = "chrony.service" - } - - // if NtpServers was configured - for _, server := range n.KubeConf.Cluster.System.NtpServers { - - serverAddr := strings.Trim(server, " \"") - if serverAddr == currentHost.GetName() || serverAddr == currentHost.GetInternalAddress() { - allowClientCmd := fmt.Sprintf(`sed -i '/#allow/ a\allow 0.0.0.0/0' %s`, chronyConfigFile) - if _, err := runtime.GetRunner().SudoCmd(allowClientCmd, false); err != nil { - return errors.Wrapf(err, "change host:%s chronyd conf failed, please check file %s", serverAddr, chronyConfigFile) - } - } - - // use internal ip to client chronyd server - for _, host := range runtime.GetAllHosts() { - if serverAddr == host.GetName() { - serverAddr = host.GetInternalAddress() - break - } - } - - checkOrAddCmd := fmt.Sprintf(`grep -q '^server %s iburst' %s||sed '1a server %s iburst' -i %s`, serverAddr, chronyConfigFile, serverAddr, chronyConfigFile) - if _, err := runtime.GetRunner().SudoCmd(checkOrAddCmd, false); err != nil { - return errors.Wrapf(err, "set ntpserver: %s failed, please check file %s", serverAddr, chronyConfigFile) - } - - } - - // if Timezone was configured - if len(n.KubeConf.Cluster.System.Timezone) > 0 { - setTimeZoneCmd := fmt.Sprintf("timedatectl set-timezone %s", n.KubeConf.Cluster.System.Timezone) - if _, err := runtime.GetRunner().SudoCmd(setTimeZoneCmd, false); err != nil { - return errors.Wrapf(err, "set timezone: %s failed", n.KubeConf.Cluster.System.Timezone) - } - - if _, err := runtime.GetRunner().SudoCmd("timedatectl set-ntp true", false); err != nil { - return errors.Wrap(err, "timedatectl set-ntp true failed") - } - } - - // ensure chronyd was enabled and work normally - if len(n.KubeConf.Cluster.System.NtpServers) > 0 || len(n.KubeConf.Cluster.System.Timezone) > 0 { - startChronyCmd := fmt.Sprintf("systemctl enable %s && systemctl restart %s", chronyService, chronyService) - if _, err := runtime.GetRunner().SudoCmd(startChronyCmd, false); err != nil { - return errors.Wrap(err, "restart chronyd failed") - } - - // tells chronyd to cancel any remaining correction that was being slewed and jump the system clock by the equivalent amount, making it correct immediately. - if _, err := runtime.GetRunner().SudoCmd("chronyc makestep > /dev/null && chronyc sources", true); err != nil { - return errors.Wrap(err, "chronyc makestep failed") - } - } - - return nil -} diff --git a/cmd/kk/pkg/bootstrap/os/templates/init_script.go b/cmd/kk/pkg/bootstrap/os/templates/init_script.go deleted file mode 100644 index 5bceb2085..000000000 --- a/cmd/kk/pkg/bootstrap/os/templates/init_script.go +++ /dev/null @@ -1,247 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "fmt" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/registry" - "text/template" - - "github.com/lithammer/dedent" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -var InitOsScriptTmpl = template.Must(template.New("initOS.sh").Parse( - dedent.Dedent(`#!/usr/bin/env bash - -# Copyright 2020 The KubeSphere Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -swapoff -a -sed -i /^[^#]*swap*/s/^/\#/g /etc/fstab - -# See https://github.com/kubernetes/website/issues/14457 -if [ -f /etc/selinux/config ]; then - sed -ri 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config -fi -# for ubuntu: sudo apt install selinux-utils -# for centos: yum install selinux-policy -if command -v setenforce &> /dev/null -then - setenforce 0 - getenforce -fi - -echo 'net.ipv4.ip_forward = 1' >> /etc/sysctl.conf -echo 'net.bridge.bridge-nf-call-arptables = 1' >> /etc/sysctl.conf -echo 'net.bridge.bridge-nf-call-ip6tables = 1' >> /etc/sysctl.conf -echo 'net.bridge.bridge-nf-call-iptables = 1' >> /etc/sysctl.conf -echo 'net.ipv4.ip_local_reserved_ports = 30000-32767' >> /etc/sysctl.conf -echo 'net.core.netdev_max_backlog = 65535' >> /etc/sysctl.conf -echo 'net.core.rmem_max = 33554432' >> /etc/sysctl.conf -echo 'net.core.wmem_max = 33554432' >> /etc/sysctl.conf -echo 'net.core.somaxconn = 32768' >> /etc/sysctl.conf -echo 'net.ipv4.tcp_max_syn_backlog = 1048576' >> /etc/sysctl.conf -echo 'net.ipv4.neigh.default.gc_thresh1 = 512' >> /etc/sysctl.conf -echo 'net.ipv4.neigh.default.gc_thresh2 = 2048' >> /etc/sysctl.conf -echo 'net.ipv4.neigh.default.gc_thresh3 = 4096' >> /etc/sysctl.conf -echo 'net.ipv4.tcp_retries2 = 15' >> /etc/sysctl.conf -echo 'net.ipv4.tcp_max_tw_buckets = 1048576' >> /etc/sysctl.conf -echo 'net.ipv4.tcp_max_orphans = 65535' >> /etc/sysctl.conf -echo 'net.ipv4.udp_rmem_min = 131072' >> /etc/sysctl.conf -echo 'net.ipv4.udp_wmem_min = 131072' >> /etc/sysctl.conf -echo 'net.ipv4.conf.all.rp_filter = 1' >> /etc/sysctl.conf -echo 'net.ipv4.conf.default.rp_filter = 1' >> /etc/sysctl.conf -echo 'net.ipv4.conf.all.arp_accept = 1' >> /etc/sysctl.conf -echo 'net.ipv4.conf.default.arp_accept = 1' >> /etc/sysctl.conf -echo 'net.ipv4.conf.all.arp_ignore = 1' >> /etc/sysctl.conf -echo 'net.ipv4.conf.default.arp_ignore = 1' >> /etc/sysctl.conf -echo 'vm.max_map_count = 262144' >> /etc/sysctl.conf -echo 'vm.swappiness = 0' >> /etc/sysctl.conf -echo 'vm.overcommit_memory = 1' >> /etc/sysctl.conf -echo 'fs.inotify.max_user_instances = 524288' >> /etc/sysctl.conf -echo 'fs.inotify.max_user_watches = 10240001' >> /etc/sysctl.conf -echo 'fs.pipe-max-size = 4194304' >> /etc/sysctl.conf -echo 'fs.aio-max-nr = 262144' >> /etc/sysctl.conf -echo 'kernel.pid_max = 65535' >> /etc/sysctl.conf -echo 'kernel.watchdog_thresh = 5' >> /etc/sysctl.conf -echo 'kernel.hung_task_timeout_secs = 5' >> /etc/sysctl.conf - -#See https://help.aliyun.com/document_detail/118806.html#uicontrol-e50-ddj-w0y -sed -r -i "s@#{0,}?net.ipv4.tcp_tw_recycle ?= ?(0|1|2)@net.ipv4.tcp_tw_recycle = 0@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.tcp_tw_reuse ?= ?(0|1)@net.ipv4.tcp_tw_reuse = 0@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.conf.all.rp_filter ?= ?(0|1|2)@net.ipv4.conf.all.rp_filter = 1@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.conf.default.rp_filter ?= ?(0|1|2)@net.ipv4.conf.default.rp_filter = 1@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.ip_forward ?= ?(0|1)@net.ipv4.ip_forward = 1@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.bridge.bridge-nf-call-arptables ?= ?(0|1)@net.bridge.bridge-nf-call-arptables = 1@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.bridge.bridge-nf-call-ip6tables ?= ?(0|1)@net.bridge.bridge-nf-call-ip6tables = 1@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.bridge.bridge-nf-call-iptables ?= ?(0|1)@net.bridge.bridge-nf-call-iptables = 1@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.ip_local_reserved_ports ?= ?([0-9]{1,}-{0,1},{0,1}){1,}@net.ipv4.ip_local_reserved_ports = 30000-32767@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?vm.max_map_count ?= ?([0-9]{1,})@vm.max_map_count = 262144@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?vm.swappiness ?= ?([0-9]{1,})@vm.swappiness = 0@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?fs.inotify.max_user_instances ?= ?([0-9]{1,})@fs.inotify.max_user_instances = 524288@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?kernel.pid_max ?= ?([0-9]{1,})@kernel.pid_max = 65535@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?vm.overcommit_memory ?= ?(0|1|2)@vm.overcommit_memory = 0@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?fs.inotify.max_user_watches ?= ?([0-9]{1,})@fs.inotify.max_user_watches = 524288@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?fs.pipe-max-size ?= ?([0-9]{1,})@fs.pipe-max-size = 4194304@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.core.netdev_max_backlog ?= ?([0-9]{1,})@net.core.netdev_max_backlog = 65535@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.core.rmem_max ?= ?([0-9]{1,})@net.core.rmem_max = 33554432@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.core.wmem_max ?= ?([0-9]{1,})@net.core.wmem_max = 33554432@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.tcp_max_syn_backlog ?= ?([0-9]{1,})@net.ipv4.tcp_max_syn_backlog = 1048576@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.neigh.default.gc_thresh1 ?= ?([0-9]{1,})@net.ipv4.neigh.default.gc_thresh1 = 512@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.neigh.default.gc_thresh2 ?= ?([0-9]{1,})@net.ipv4.neigh.default.gc_thresh2 = 2048@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.neigh.default.gc_thresh3 ?= ?([0-9]{1,})@net.ipv4.neigh.default.gc_thresh3 = 4096@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.core.somaxconn ?= ?([0-9]{1,})@net.core.somaxconn = 32768@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.conf.eth0.arp_accept ?= ?(0|1)@net.ipv4.conf.eth0.arp_accept = 1@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?fs.aio-max-nr ?= ?([0-9]{1,})@fs.aio-max-nr = 262144@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.tcp_retries2 ?= ?([0-9]{1,})@net.ipv4.tcp_retries2 = 15@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.tcp_max_tw_buckets ?= ?([0-9]{1,})@net.ipv4.tcp_max_tw_buckets = 1048576@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.tcp_max_orphans ?= ?([0-9]{1,})@net.ipv4.tcp_max_orphans = 65535@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.udp_rmem_min ?= ?([0-9]{1,})@net.ipv4.udp_rmem_min = 131072@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.udp_wmem_min ?= ?([0-9]{1,})@net.ipv4.udp_wmem_min = 131072@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.conf.all.arp_ignore ?= ??(0|1|2)@net.ipv4.conf.all.arp_ignore = 1@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.conf.default.arp_ignore ?= ??(0|1|2)@net.ipv4.conf.default.arp_ignore = 1@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?kernel.watchdog_thresh ?= ?([0-9]{1,})@kernel.watchdog_thresh = 5@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?kernel.hung_task_timeout_secs ?= ?([0-9]{1,})@kernel.hung_task_timeout_secs = 5@g" /etc/sysctl.conf - -tmpfile="$$.tmp" -awk ' !x[$0]++{print > "'$tmpfile'"}' /etc/sysctl.conf -mv $tmpfile /etc/sysctl.conf - -# ulimit -echo "* soft nofile 1048576" >> /etc/security/limits.conf -echo "* hard nofile 1048576" >> /etc/security/limits.conf -echo "* soft nproc 65536" >> /etc/security/limits.conf -echo "* hard nproc 65536" >> /etc/security/limits.conf -echo "* soft memlock unlimited" >> /etc/security/limits.conf -echo "* hard memlock unlimited" >> /etc/security/limits.conf - -sed -r -i "s@#{0,}?\* soft nofile ?([0-9]{1,})@\* soft nofile 1048576@g" /etc/security/limits.conf -sed -r -i "s@#{0,}?\* hard nofile ?([0-9]{1,})@\* hard nofile 1048576@g" /etc/security/limits.conf -sed -r -i "s@#{0,}?\* soft nproc ?([0-9]{1,})@\* soft nproc 65536@g" /etc/security/limits.conf -sed -r -i "s@#{0,}?\* hard nproc ?([0-9]{1,})@\* hard nproc 65536@g" /etc/security/limits.conf -sed -r -i "s@#{0,}?\* soft memlock ?([0-9]{1,}([TGKM]B){0,1}|unlimited)@\* soft memlock unlimited@g" /etc/security/limits.conf -sed -r -i "s@#{0,}?\* hard memlock ?([0-9]{1,}([TGKM]B){0,1}|unlimited)@\* hard memlock unlimited@g" /etc/security/limits.conf - -tmpfile="$$.tmp" -awk ' !x[$0]++{print > "'$tmpfile'"}' /etc/security/limits.conf -mv $tmpfile /etc/security/limits.conf - -systemctl stop firewalld 1>/dev/null 2>/dev/null -systemctl disable firewalld 1>/dev/null 2>/dev/null -systemctl stop ufw 1>/dev/null 2>/dev/null -systemctl disable ufw 1>/dev/null 2>/dev/null - -modinfo br_netfilter > /dev/null 2>&1 -if [ $? -eq 0 ]; then - modprobe br_netfilter - mkdir -p /etc/modules-load.d - echo 'br_netfilter' > /etc/modules-load.d/kubekey-br_netfilter.conf -fi - -modinfo overlay > /dev/null 2>&1 -if [ $? -eq 0 ]; then - modprobe overlay - echo 'overlay' >> /etc/modules-load.d/kubekey-br_netfilter.conf -fi - -modprobe ip_vs -modprobe ip_vs_rr -modprobe ip_vs_wrr -modprobe ip_vs_sh - -cat > /etc/modules-load.d/kube_proxy-ipvs.conf << EOF -ip_vs -ip_vs_rr -ip_vs_wrr -ip_vs_sh -EOF - -modprobe nf_conntrack_ipv4 1>/dev/null 2>/dev/null -if [ $? -eq 0 ]; then - echo 'nf_conntrack_ipv4' > /etc/modules-load.d/kube_proxy-ipvs.conf -else - modprobe nf_conntrack - echo 'nf_conntrack' > /etc/modules-load.d/kube_proxy-ipvs.conf -fi -sysctl -p - -sed -i ':a;$!{N;ba};s@# kubekey hosts BEGIN.*# kubekey hosts END@@' /etc/hosts -sed -i '/^$/N;/\n$/N;//D' /etc/hosts - -cat >>/etc/hosts< /proc/sys/vm/drop_caches - -# Make sure the iptables utility doesn't use the nftables backend. -update-alternatives --set iptables /usr/sbin/iptables-legacy >/dev/null 2>&1 || true -update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy >/dev/null 2>&1 || true -update-alternatives --set arptables /usr/sbin/arptables-legacy >/dev/null 2>&1 || true -update-alternatives --set ebtables /usr/sbin/ebtables-legacy >/dev/null 2>&1 || true - - `))) - -func GenerateHosts(runtime connector.ModuleRuntime, kubeConf *common.KubeConf) []string { - var lbHost string - var hostsList []string - - if kubeConf.Cluster.ControlPlaneEndpoint.Address != "" { - lbHost = fmt.Sprintf("%s %s", kubeConf.Cluster.ControlPlaneEndpoint.Address, kubeConf.Cluster.ControlPlaneEndpoint.Domain) - } - - for _, host := range runtime.GetAllHosts() { - if host.GetName() != "" { - hostsList = append(hostsList, fmt.Sprintf("%s %s.%s %s", - host.GetInternalAddress(), - host.GetName(), - kubeConf.Cluster.Kubernetes.ClusterName, - host.GetName())) - } - } - - if len(runtime.GetHostsByRole(common.Registry)) > 0 { - if kubeConf.Cluster.Registry.PrivateRegistry != "" { - hostsList = append(hostsList, fmt.Sprintf("%s %s", runtime.GetHostsByRole(common.Registry)[0].GetInternalAddress(), kubeConf.Cluster.Registry.PrivateRegistry)) - } else { - hostsList = append(hostsList, fmt.Sprintf("%s %s", runtime.GetHostsByRole(common.Registry)[0].GetInternalAddress(), registry.RegistryCertificateBaseName)) - } - - } - - hostsList = append(hostsList, lbHost) - return hostsList -} diff --git a/cmd/kk/pkg/bootstrap/precheck/const.go b/cmd/kk/pkg/bootstrap/precheck/const.go deleted file mode 100644 index 71d5db20c..000000000 --- a/cmd/kk/pkg/bootstrap/precheck/const.go +++ /dev/null @@ -1,59 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package precheck - -const ( - // command software - sudo = "sudo" - curl = "curl" - openssl = "openssl" - ebtables = "ebtables" - socat = "socat" - ipset = "ipset" - ipvsadm = "ipvsadm" - conntrack = "conntrack" - chrony = "chronyd" - docker = "docker" - containerd = "containerd" - showmount = "showmount" - rbd = "rbd" - glusterfs = "glusterfs" - - // extra command tools - nfs = "nfs" - ceph = "ceph" - - UnknownVersion = "UnknownVersion" -) - -// defines the base software to be checked. -var baseSoftware = []string{ - sudo, - curl, - openssl, - ebtables, - socat, - ipset, - ipvsadm, - conntrack, - chrony, - docker, - containerd, - showmount, - rbd, - glusterfs, -} diff --git a/cmd/kk/pkg/bootstrap/precheck/module.go b/cmd/kk/pkg/bootstrap/precheck/module.go deleted file mode 100644 index df6583cdd..000000000 --- a/cmd/kk/pkg/bootstrap/precheck/module.go +++ /dev/null @@ -1,171 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package precheck - -import ( - "time" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" -) - -type GreetingsModule struct { - module.BaseTaskModule -} - -func (h *GreetingsModule) Init() { - h.Name = "GreetingsModule" - h.Desc = "Greetings" - - var timeout int64 - for _, v := range h.Runtime.GetAllHosts() { - timeout += v.GetTimeout() - } - - hello := &task.RemoteTask{ - Name: "Greetings", - Desc: "Greetings", - Hosts: h.Runtime.GetAllHosts(), - Action: new(GreetingsTask), - Parallel: true, - Timeout: time.Duration(timeout) * time.Second, - } - - h.Tasks = []task.Interface{ - hello, - } -} - -type NodePreCheckModule struct { - common.KubeModule - Skip bool -} - -func (n *NodePreCheckModule) IsSkip() bool { - return n.Skip -} - -func (n *NodePreCheckModule) Init() { - n.Name = "NodePreCheckModule" - n.Desc = "Do pre-check on cluster nodes" - - preCheck := &task.RemoteTask{ - Name: "NodePreCheck", - Desc: "A pre-check on nodes", - Hosts: n.Runtime.GetAllHosts(), - //Prepare: &prepare.FastPrepare{ - // Inject: func(runtime connector.Runtime) (bool, error) { - // if len(n.Runtime.GetHostsByRole(common.ETCD))%2 == 0 { - // logger.Log.Error("The number of etcd is even. Please configure it to be odd.") - // return false, errors.New("the number of etcd is even") - // } - // return true, nil - // }}, - Action: new(NodePreCheck), - Parallel: true, - } - - n.Tasks = []task.Interface{ - preCheck, - } -} - -type ClusterPreCheckModule struct { - common.KubeModule -} - -func (c *ClusterPreCheckModule) Init() { - c.Name = "ClusterPreCheckModule" - c.Desc = "Do pre-check on cluster" - - getKubeConfig := &task.RemoteTask{ - Name: "GetKubeConfig", - Desc: "Get KubeConfig file", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(GetKubeConfig), - Parallel: true, - } - - getAllNodesK8sVersion := &task.RemoteTask{ - Name: "GetAllNodesK8sVersion", - Desc: "Get all nodes Kubernetes version", - Hosts: c.Runtime.GetHostsByRole(common.K8s), - Action: new(GetAllNodesK8sVersion), - Parallel: true, - } - - calculateMinK8sVersion := &task.RemoteTask{ - Name: "CalculateMinK8sVersion", - Desc: "Calculate min Kubernetes version", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(CalculateMinK8sVersion), - Parallel: true, - } - - checkDesiredK8sVersion := &task.RemoteTask{ - Name: "CheckDesiredK8sVersion", - Desc: "Check desired Kubernetes version", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(CheckDesiredK8sVersion), - Parallel: true, - } - - ksVersionCheck := &task.RemoteTask{ - Name: "KsVersionCheck", - Desc: "Check KubeSphere version", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(KsVersionCheck), - Parallel: true, - } - - dependencyCheck := &task.RemoteTask{ - Name: "DependencyCheck", - Desc: "Check dependency matrix for KubeSphere and Kubernetes", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(KubeSphereExist), - }, - Action: new(DependencyCheck), - Parallel: true, - } - - getKubernetesNodesStatus := &task.RemoteTask{ - Name: "GetKubernetesNodesStatus", - Desc: "Get kubernetes nodes status", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(GetKubernetesNodesStatus), - Parallel: true, - } - - c.Tasks = []task.Interface{ - getKubeConfig, - getAllNodesK8sVersion, - calculateMinK8sVersion, - checkDesiredK8sVersion, - ksVersionCheck, - dependencyCheck, - getKubernetesNodesStatus, - } -} diff --git a/cmd/kk/pkg/bootstrap/precheck/prepares.go b/cmd/kk/pkg/bootstrap/precheck/prepares.go deleted file mode 100644 index 53c2e10af..000000000 --- a/cmd/kk/pkg/bootstrap/precheck/prepares.go +++ /dev/null @@ -1,39 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package precheck - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type KubeSphereExist struct { - common.KubePrepare -} - -func (k *KubeSphereExist) PreCheck(runtime connector.Runtime) (bool, error) { - currentKsVersion, ok := k.PipelineCache.GetMustString(common.KubeSphereVersion) - if !ok { - return false, errors.New("get current KubeSphere version failed by pipeline cache") - } - if currentKsVersion != "" { - return true, nil - } - return false, nil -} diff --git a/cmd/kk/pkg/bootstrap/precheck/tasks.go b/cmd/kk/pkg/bootstrap/precheck/tasks.go deleted file mode 100644 index c6d60cb71..000000000 --- a/cmd/kk/pkg/bootstrap/precheck/tasks.go +++ /dev/null @@ -1,336 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package precheck - -import ( - "fmt" - "regexp" - "strings" - - "github.com/pkg/errors" - versionutil "k8s.io/apimachinery/pkg/util/version" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere" -) - -type GreetingsTask struct { - action.BaseAction -} - -func (h *GreetingsTask) Execute(runtime connector.Runtime) error { - hello, err := runtime.GetRunner().SudoCmd("echo 'Greetings, KubeKey!'", false) - if err != nil { - return err - } - logger.Log.Messagef(runtime.RemoteHost().GetName(), hello) - return nil -} - -type NodePreCheck struct { - common.KubeAction -} - -func (n *NodePreCheck) Execute(runtime connector.Runtime) error { - var results = make(map[string]string) - results["name"] = runtime.RemoteHost().GetName() - for _, software := range baseSoftware { - var ( - cmd string - ) - - switch software { - case docker: - cmd = "docker version --format '{{.Server.Version}}'" - case containerd: - cmd = "containerd --version | cut -d ' ' -f 3" - default: - cmd = fmt.Sprintf("which %s", software) - } - - switch software { - case sudo: - // sudo skip sudo prefix - default: - cmd = connector.SudoPrefix(cmd) - } - - res, err := runtime.GetRunner().Cmd(cmd, false) - switch software { - case showmount: - software = nfs - case rbd: - software = ceph - case glusterfs: - software = glusterfs - } - if err != nil || strings.Contains(res, "not found") { - results[software] = "" - } else { - // software in path - if strings.Contains(res, "bin/") { - results[software] = "y" - } else { - // get software version, e.g. docker, containerd, etc. - results[software] = res - } - } - } - - output, err := runtime.GetRunner().Cmd("date +\"%Z %H:%M:%S\"", false) - if err != nil { - results["time"] = "" - } else { - results["time"] = strings.TrimSpace(output) - } - - host := runtime.RemoteHost() - if res, ok := host.GetCache().Get(common.NodePreCheck); ok { - m := res.(map[string]string) - m = results - host.GetCache().Set(common.NodePreCheck, m) - } else { - host.GetCache().Set(common.NodePreCheck, results) - } - return nil -} - -type GetKubeConfig struct { - common.KubeAction -} - -func (g *GetKubeConfig) Execute(runtime connector.Runtime) error { - if exist, err := runtime.GetRunner().FileExist("$HOME/.kube/config"); err != nil { - return err - } else { - if exist { - return nil - } else { - if exist, err := runtime.GetRunner().FileExist("/etc/kubernetes/admin.conf"); err != nil { - return err - } else { - if exist { - if _, err := runtime.GetRunner().Cmd("mkdir -p $HOME/.kube", false); err != nil { - return err - } - if _, err := runtime.GetRunner().SudoCmd("cp /etc/kubernetes/admin.conf $HOME/.kube/config", false); err != nil { - return err - } - userId, err := runtime.GetRunner().Cmd("echo $(id -u)", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get user id failed") - } - - userGroupId, err := runtime.GetRunner().Cmd("echo $(id -g)", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get user group id failed") - } - - chownKubeConfig := fmt.Sprintf("chown -R %s:%s $HOME/.kube", userId, userGroupId) - if _, err := runtime.GetRunner().SudoCmd(chownKubeConfig, false); err != nil { - return errors.Wrap(errors.WithStack(err), "chown user kube config failed") - } - } - } - } - } - return errors.New("kube config not found") -} - -type GetAllNodesK8sVersion struct { - common.KubeAction -} - -func (g *GetAllNodesK8sVersion) Execute(runtime connector.Runtime) error { - var nodeK8sVersion string - kubeletVersionInfo, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubelet --version", false) - if err != nil { - return errors.Wrap(err, "get current kubelet version failed") - } - nodeK8sVersion = strings.Split(kubeletVersionInfo, " ")[1] - - host := runtime.RemoteHost() - host.GetCache().Set(common.NodeK8sVersion, nodeK8sVersion) - - if host.IsRole(common.Master) { - apiserverVersion, err := runtime.GetRunner().SudoCmd( - "cat /etc/kubernetes/manifests/kube-apiserver.yaml | grep 'image:' | rev | cut -d ':' -f1 | rev", - false) - if err != nil { - return errors.Wrap(err, "get current kube-apiserver version failed") - } - - apiserverSemanticVersion, err := versionutil.ParseSemantic(apiserverVersion) - if err != nil { - return errors.Wrap(err, "parse kube-apiserver version failed") - } - - kubeletSemanticVersion, err := versionutil.ParseSemantic(nodeK8sVersion) - if err != nil { - return errors.Wrap(err, "parse kubelet version failed") - } - - if apiserverSemanticVersion.LessThan(kubeletSemanticVersion) { - host.GetCache().Set(common.NodeK8sVersion, apiserverVersion) - } - } - - return nil -} - -type CalculateMinK8sVersion struct { - common.KubeAction -} - -func (g *CalculateMinK8sVersion) Execute(runtime connector.Runtime) error { - versionList := make([]*versionutil.Version, 0, len(runtime.GetHostsByRole(common.K8s))) - for _, host := range runtime.GetHostsByRole(common.K8s) { - version, ok := host.GetCache().GetMustString(common.NodeK8sVersion) - if !ok { - return errors.Errorf("get node %s Kubernetes version failed by host cache", host.GetName()) - } - if versionObj, err := versionutil.ParseSemantic(version); err != nil { - return errors.Wrap(err, "parse node version failed") - } else { - versionList = append(versionList, versionObj) - } - } - - minVersion := versionList[0] - for _, version := range versionList { - if !minVersion.LessThan(version) { - minVersion = version - } - } - g.PipelineCache.Set(common.K8sVersion, fmt.Sprintf("v%s", minVersion)) - return nil -} - -type CheckDesiredK8sVersion struct { - common.KubeAction -} - -func (k *CheckDesiredK8sVersion) Execute(_ connector.Runtime) error { - if ok := kubernetes.VersionSupport(k.KubeConf.Cluster.Kubernetes.Version); !ok { - return errors.New(fmt.Sprintf("does not support upgrade to Kubernetes %s", - k.KubeConf.Cluster.Kubernetes.Version)) - } - k.PipelineCache.Set(common.DesiredK8sVersion, k.KubeConf.Cluster.Kubernetes.Version) - return nil -} - -type KsVersionCheck struct { - common.KubeAction -} - -func (k *KsVersionCheck) Execute(runtime connector.Runtime) error { - ksVersionStr, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl get deploy -n kubesphere-system ks-console -o jsonpath='{.metadata.labels.version}'", - false) - if err != nil { - if k.KubeConf.Cluster.KubeSphere.Enabled { - return errors.Wrap(err, "get kubeSphere version failed") - } else { - ksVersionStr = "" - } - } - - ccKsVersionStr, ccErr := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl get ClusterConfiguration ks-installer -n kubesphere-system -o jsonpath='{.metadata.labels.version}'", - false) - - if ccErr == nil && versionutil.MustParseSemantic(ccKsVersionStr).AtLeast(versionutil.MustParseSemantic("v3.1.0")) { - ksVersionStr = ccKsVersionStr - } - - k.PipelineCache.Set(common.KubeSphereVersion, ksVersionStr) - return nil -} - -type DependencyCheck struct { - common.KubeAction -} - -func (d *DependencyCheck) Execute(_ connector.Runtime) error { - currentKsVersion, ok := d.PipelineCache.GetMustString(common.KubeSphereVersion) - if !ok { - return errors.New("get current KubeSphere version failed by pipeline cache") - } - desiredVersion := d.KubeConf.Cluster.KubeSphere.Version - - if d.KubeConf.Cluster.KubeSphere.Enabled { - var version string - if latest, ok := kubesphere.LatestRelease(desiredVersion); ok { - version = latest.Version - } else if ks, ok := kubesphere.DevRelease(desiredVersion); ok { - version = ks.Version - } else { - r := regexp.MustCompile("v(\\d+\\.)?(\\d+\\.)?(\\*|\\d+)") - version = r.FindString(desiredVersion) - } - - ksInstaller, ok := kubesphere.VersionMap[version] - if !ok { - return errors.New(fmt.Sprintf("Unsupported version: %s", desiredVersion)) - } - - if currentKsVersion != desiredVersion { - if ok := ksInstaller.UpgradeSupport(currentKsVersion); !ok { - return errors.New(fmt.Sprintf("Unsupported upgrade plan: %s to %s", currentKsVersion, desiredVersion)) - } - } - - if ok := ksInstaller.K8sSupport(d.KubeConf.Cluster.Kubernetes.Version); !ok { - return errors.New(fmt.Sprintf("KubeSphere %s does not support running on Kubernetes %s", - version, d.KubeConf.Cluster.Kubernetes.Version)) - } - } else { - ksInstaller, ok := kubesphere.VersionMap[currentKsVersion] - if !ok { - return errors.New(fmt.Sprintf("Unsupported version: %s", currentKsVersion)) - } - - if ok := ksInstaller.K8sSupport(d.KubeConf.Cluster.Kubernetes.Version); !ok { - return errors.New(fmt.Sprintf("KubeSphere %s does not support running on Kubernetes %s", - currentKsVersion, d.KubeConf.Cluster.Kubernetes.Version)) - } - } - return nil -} - -type GetKubernetesNodesStatus struct { - common.KubeAction -} - -func (g *GetKubernetesNodesStatus) Execute(runtime connector.Runtime) error { - nodeStatus, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl get node -o wide", false) - if err != nil { - return err - } - g.PipelineCache.Set(common.ClusterNodeStatus, nodeStatus) - - cri, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl get node -o jsonpath=\"{.items[*].status.nodeInfo.containerRuntimeVersion}\"", false) - if err != nil { - return err - } - g.PipelineCache.Set(common.ClusterNodeCRIRuntimes, cri) - return nil -} diff --git a/cmd/kk/pkg/bootstrap/registry/certs.go b/cmd/kk/pkg/bootstrap/registry/certs.go deleted file mode 100644 index 8b5918477..000000000 --- a/cmd/kk/pkg/bootstrap/registry/certs.go +++ /dev/null @@ -1,146 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package registry - -import ( - "crypto/x509" - "fmt" - "net" - "path/filepath" - "strings" - - "github.com/pkg/errors" - "k8s.io/client-go/util/cert" - certutil "k8s.io/client-go/util/cert" - netutils "k8s.io/utils/net" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils/certs" -) - -const ( - RegistryCertificateBaseName = "dockerhub.kubekey.local" - LocalCertsDir = "localCertsDir" - CertsFileList = "certsFileList" -) - -// KubekeyCertRegistryCA is the definition of the root CA used by the hosted etcd server. -func KubekeyCertRegistryCA() *certs.KubekeyCert { - return &certs.KubekeyCert{ - Name: "registry-ca", - LongName: "self-signed CA to provision identities for registry", - BaseName: "ca", - Config: certs.CertConfig{ - Config: certutil.Config{ - CommonName: "registry-ca", - }, - }, - } -} - -// KubekeyCertRegistryServer is the definition of the cert for etcd admin. -func KubekeyCertRegistryServer(baseName string, altNames *certutil.AltNames) *certs.KubekeyCert { - return &certs.KubekeyCert{ - Name: "registry-server", - LongName: "certificate for registry server", - BaseName: baseName, - CAName: "registry-ca", - Config: certs.CertConfig{ - Config: certutil.Config{ - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, - AltNames: *altNames, - CommonName: baseName, - }, - }, - } -} - -type FetchCerts struct { - common.KubeAction -} - -func (f *FetchCerts) Execute(runtime connector.Runtime) error { - src := "/etc/ssl/registry/ssl" - dst := fmt.Sprintf("%s/pki/registry", runtime.GetWorkDir()) - - certs, err := runtime.GetRunner().SudoCmd("ls /etc/ssl/registry/ssl/ | grep .pem", false) - if err != nil { - return nil - } - - certsList := strings.Split(certs, "\r\n") - if len(certsList) > 0 { - for _, cert := range certsList { - if err := runtime.GetRunner().Fetch(filepath.Join(dst, cert), filepath.Join(src, cert)); err != nil { - return errors.Wrap(err, fmt.Sprintf("Fetch %s failed", filepath.Join(src, cert))) - } - } - } - - return nil -} - -type GenerateCerts struct { - common.KubeAction -} - -func (g *GenerateCerts) Execute(runtime connector.Runtime) error { - - pkiPath := fmt.Sprintf("%s/pki/registry", runtime.GetWorkDir()) - - var altName cert.AltNames - - dnsList := []string{"localhost", RegistryCertificateBaseName} - ipList := []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback} - - for _, h := range runtime.GetHostsByRole(common.Registry) { - dnsList = append(dnsList, h.GetName()) - ipList = append(ipList, netutils.ParseIPSloppy(h.GetInternalAddress())) - } - altName.DNSNames = dnsList - altName.IPs = ipList - - files := []string{"ca.pem", "ca-key.pem", fmt.Sprintf("%s.pem", g.KubeConf.Cluster.Registry.PrivateRegistry), fmt.Sprintf("%s-key.pem", g.KubeConf.Cluster.Registry.PrivateRegistry)} - - // CA - certsList := []*certs.KubekeyCert{KubekeyCertRegistryCA()} - - // Certs - certsList = append(certsList, KubekeyCertRegistryServer(g.KubeConf.Cluster.Registry.PrivateRegistry, &altName)) - - var lastCACert *certs.KubekeyCert - for _, c := range certsList { - if c.CAName == "" { - err := certs.GenerateCA(c, pkiPath, g.KubeConf) - if err != nil { - return err - } - lastCACert = c - } else { - err := certs.GenerateCerts(c, lastCACert, pkiPath, g.KubeConf) - if err != nil { - return err - } - } - } - - g.ModuleCache.Set(LocalCertsDir, pkiPath) - g.ModuleCache.Set(CertsFileList, files) - - return nil -} diff --git a/cmd/kk/pkg/bootstrap/registry/module.go b/cmd/kk/pkg/bootstrap/registry/module.go deleted file mode 100644 index 284b88bac..000000000 --- a/cmd/kk/pkg/bootstrap/registry/module.go +++ /dev/null @@ -1,281 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package registry - -import ( - "fmt" - "path/filepath" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/registry/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container" - docker_template "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -type RegistryCertsModule struct { - common.KubeModule - Skip bool -} - -func (p *RegistryCertsModule) IsSkip() bool { - return p.Skip -} - -func (i *RegistryCertsModule) Init() { - i.Name = "InitRegistryModule" - i.Desc = "Init a local registry" - - fetchCerts := &task.RemoteTask{ - Name: "FetchRegistryCerts", - Desc: "Fetch registry certs", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Prepare: new(FirstRegistryNode), - Action: new(FetchCerts), - Parallel: false, - } - - generateCerts := &task.LocalTask{ - Name: "GenerateRegistryCerts", - Desc: "Generate registry Certs", - Action: new(GenerateCerts), - } - - syncCertsFile := &task.RemoteTask{ - Name: "SyncCertsFile", - Desc: "Synchronize certs file", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Action: new(SyncCertsFile), - Parallel: true, - Retry: 1, - } - - syncCertsToAllNodes := &task.RemoteTask{ - Name: "SyncCertsFileToAllNodes", - Desc: "Synchronize certs file to all nodes", - Hosts: i.Runtime.GetAllHosts(), - Action: new(SyncCertsToAllNodes), - Parallel: true, - Retry: 1, - } - - i.Tasks = []task.Interface{ - fetchCerts, - generateCerts, - syncCertsFile, - syncCertsToAllNodes, - } - -} - -type InstallRegistryModule struct { - common.KubeModule -} - -func (i *InstallRegistryModule) Init() { - i.Name = "InstallRegistryModule" - i.Desc = "Install local registry" - - switch i.KubeConf.Cluster.Registry.Type { - case common.Harbor: - i.Tasks = InstallHarbor(i) - default: - i.Tasks = InstallRegistry(i) - } -} - -func InstallRegistry(i *InstallRegistryModule) []task.Interface { - installRegistryBinary := &task.RemoteTask{ - Name: "InstallRegistryBinary", - Desc: "Install registry binary", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Action: new(InstallRegistryBinary), - Parallel: true, - Retry: 1, - } - - generateRegistryService := &task.RemoteTask{ - Name: "GenerateRegistryService", - Desc: "Generate registry service", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Action: &action.Template{ - Template: templates.RegistryServiceTempl, - Dst: "/etc/systemd/system/registry.service", - }, - Parallel: true, - Retry: 1, - } - - generateRegistryConfig := &task.RemoteTask{ - Name: "GenerateRegistryConfig", - Desc: "Generate registry config", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Action: &action.Template{ - Template: templates.RegistryConfigTempl, - Dst: "/etc/kubekey/registry/config.yaml", - Data: util.Data{ - "Certificate": fmt.Sprintf("%s.pem", i.KubeConf.Cluster.Registry.PrivateRegistry), - "Key": fmt.Sprintf("%s-key.pem", i.KubeConf.Cluster.Registry.PrivateRegistry), - }, - }, - Parallel: true, - Retry: 1, - } - - startRegistryService := &task.RemoteTask{ - Name: "StartRegistryService", - Desc: "Start registry service", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Action: new(StartRegistryService), - Parallel: true, - Retry: 1, - } - - return []task.Interface{ - installRegistryBinary, - generateRegistryService, - generateRegistryConfig, - startRegistryService, - } -} - -func InstallHarbor(i *InstallRegistryModule) []task.Interface { - // Install docker - syncBinaries := &task.RemoteTask{ - Name: "SyncDockerBinaries", - Desc: "Sync docker binaries", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Prepare: &prepare.PrepareCollection{ - &container.DockerExist{Not: true}, - }, - Action: new(container.SyncDockerBinaries), - Parallel: true, - Retry: 2, - } - - generateDockerService := &task.RemoteTask{ - Name: "GenerateDockerService", - Desc: "Generate docker service", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Prepare: &prepare.PrepareCollection{ - &container.DockerExist{Not: true}, - }, - Action: &action.Template{ - Template: docker_template.DockerService, - Dst: filepath.Join("/etc/systemd/system", docker_template.DockerService.Name()), - }, - Parallel: true, - } - - generateDockerConfig := &task.RemoteTask{ - Name: "GenerateDockerConfig", - Desc: "Generate docker config", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Prepare: &prepare.PrepareCollection{ - &container.DockerExist{Not: true}, - }, - Action: &action.Template{ - Template: docker_template.DockerConfig, - Dst: filepath.Join("/etc/docker/", docker_template.DockerConfig.Name()), - Data: util.Data{ - "Mirrors": docker_template.Mirrors(i.KubeConf), - "InsecureRegistries": docker_template.InsecureRegistries(i.KubeConf), - }, - }, - Parallel: true, - } - - enableDocker := &task.RemoteTask{ - Name: "EnableDocker", - Desc: "Enable docker", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Prepare: &prepare.PrepareCollection{ - &container.DockerExist{Not: true}, - }, - Action: new(container.EnableDocker), - Parallel: true, - } - - // Install docker compose - installDockerCompose := &task.RemoteTask{ - Name: "InstallDockerCompose", - Desc: "Install docker compose", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Action: new(InstallDockerCompose), - Parallel: true, - Retry: 2, - } - - // Install Harbor - syncHarborPackage := &task.RemoteTask{ - Name: "SyncHarborPackage", - Desc: "Sync harbor package", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Action: new(SyncHarborPackage), - Parallel: true, - Retry: 2, - } - - // generate Harbor Systemd - generateHarborService := &task.RemoteTask{ - Name: "GenerateHarborService", - Desc: "Generate harbor service", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Action: &action.Template{ - Template: templates.HarborServiceTempl, - Dst: "/etc/systemd/system/harbor.service", - Data: util.Data{ - "Harbor_install_path": "/opt/harbor", - }, - }, - Parallel: true, - Retry: 1, - } - - generateHarborConfig := &task.RemoteTask{ - Name: "GenerateHarborConfig", - Desc: "Generate harbor config", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Action: new(GenerateHarborConfig), - Parallel: true, - Retry: 1, - } - - startHarbor := &task.RemoteTask{ - Name: "StartHarbor", - Desc: "start harbor", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Action: new(StartHarbor), - Parallel: true, - Retry: 2, - } - - return []task.Interface{ - syncBinaries, - generateDockerService, - generateDockerConfig, - enableDocker, - installDockerCompose, - syncHarborPackage, - generateHarborService, - generateHarborConfig, - startHarbor, - } -} diff --git a/cmd/kk/pkg/bootstrap/registry/prepares.go b/cmd/kk/pkg/bootstrap/registry/prepares.go deleted file mode 100644 index 6db266768..000000000 --- a/cmd/kk/pkg/bootstrap/registry/prepares.go +++ /dev/null @@ -1,34 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package registry - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type FirstRegistryNode struct { - common.KubePrepare - Not bool -} - -func (f *FirstRegistryNode) PreCheck(runtime connector.Runtime) (bool, error) { - if runtime.GetHostsByRole(common.Registry)[0].GetName() == runtime.RemoteHost().GetName() { - return !f.Not, nil - } - return f.Not, nil -} diff --git a/cmd/kk/pkg/bootstrap/registry/tasks.go b/cmd/kk/pkg/bootstrap/registry/tasks.go deleted file mode 100644 index e3b9e62a4..000000000 --- a/cmd/kk/pkg/bootstrap/registry/tasks.go +++ /dev/null @@ -1,256 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package registry - -import ( - "fmt" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/registry/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "path/filepath" - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" -) - -type SyncCertsFile struct { - common.KubeAction -} - -func (s *SyncCertsFile) Execute(runtime connector.Runtime) error { - localCertsDir, ok := s.ModuleCache.Get(LocalCertsDir) - if !ok { - return errors.New("get registry local certs dir by module cache failed") - } - files, ok := s.ModuleCache.Get(CertsFileList) - if !ok { - return errors.New("get registry certs file list by module cache failed") - } - dir := localCertsDir.(string) - fileList := files.([]string) - - for _, fileName := range fileList { - if err := runtime.GetRunner().SudoScp(filepath.Join(dir, fileName), filepath.Join(common.RegistryCertDir, fileName)); err != nil { - return errors.Wrap(errors.WithStack(err), "scp registry certs file failed") - } - } - - return nil -} - -type SyncCertsToAllNodes struct { - common.KubeAction -} - -func (s *SyncCertsToAllNodes) Execute(runtime connector.Runtime) error { - localCertsDir, ok := s.ModuleCache.Get(LocalCertsDir) - if !ok { - return errors.New("get registry local certs dir by module cache failed") - } - files, ok := s.ModuleCache.Get(CertsFileList) - if !ok { - return errors.New("get registry certs file list by module cache failed") - } - dir := localCertsDir.(string) - fileList := files.([]string) - - for _, fileName := range fileList { - var dstFileName string - switch fileName { - case "ca.pem": - dstFileName = "ca.crt" - case "ca-key.pem": - continue - default: - if strings.HasSuffix(fileName, "-key.pem") { - dstFileName = strings.Replace(fileName, "-key.pem", ".key", -1) - } else { - dstFileName = strings.Replace(fileName, ".pem", ".cert", -1) - } - } - - if err := runtime.GetRunner().SudoScp(filepath.Join(dir, fileName), filepath.Join(filepath.Join("/etc/docker/certs.d", s.KubeConf.Cluster.Registry.PrivateRegistry), dstFileName)); err != nil { - return errors.Wrap(errors.WithStack(err), "scp registry certs file to /etc/docker/certs.d/ failed") - } - - if err := runtime.GetRunner().SudoScp(filepath.Join(dir, fileName), filepath.Join(common.RegistryCertDir, dstFileName)); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("scp registry certs file to %s failed", common.RegistryCertDir)) - } - } - - return nil -} - -type InstallRegistryBinary struct { - common.KubeAction -} - -func (g *InstallRegistryBinary) Execute(runtime connector.Runtime) error { - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - binariesMapObj, ok := g.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) - if !ok { - return errors.New("get KubeBinary by pipeline cache failed") - } - binariesMap := binariesMapObj.(map[string]*files.KubeBinary) - - registry, ok := binariesMap[common.Registry] - if !ok { - return errors.New("get KubeBinary key registry by pipeline cache failed") - } - - dst := filepath.Join(common.TmpDir, registry.FileName) - if err := runtime.GetRunner().Scp(registry.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), "sync etcd tar.gz failed") - } - - installCmd := fmt.Sprintf("tar -zxf %s && mv -f registry /usr/local/bin/ && chmod +x /usr/local/bin/registry", dst) - if _, err := runtime.GetRunner().SudoCmd(installCmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "install etcd binaries failed") - } - return nil -} - -type StartRegistryService struct { - common.KubeAction -} - -func (g *StartRegistryService) Execute(runtime connector.Runtime) error { - installCmd := "systemctl daemon-reload && systemctl enable registry && systemctl restart registry" - if _, err := runtime.GetRunner().SudoCmd(installCmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "start registry service failed") - } - - fmt.Println() - fmt.Println(fmt.Sprintf("Local image registry created successfully. Address: %s", g.KubeConf.Cluster.Registry.PrivateRegistry)) - fmt.Println() - - return nil -} - -type InstallDockerCompose struct { - common.KubeAction -} - -func (g *InstallDockerCompose) Execute(runtime connector.Runtime) error { - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - binariesMapObj, ok := g.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) - if !ok { - return errors.New("get KubeBinary by pipeline cache failed") - } - binariesMap := binariesMapObj.(map[string]*files.KubeBinary) - - compose, ok := binariesMap[common.DockerCompose] - if !ok { - return errors.New("get KubeBinary key docker-compose by pipeline cache failed") - } - - dst := filepath.Join(common.TmpDir, compose.FileName) - if err := runtime.GetRunner().Scp(compose.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), "sync docker-compose failed") - } - - installCmd := fmt.Sprintf("mv -f %s /usr/local/bin/docker-compose && chmod +x /usr/local/bin/docker-compose", dst) - if _, err := runtime.GetRunner().SudoCmd(installCmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "install dokcer-compose failed") - } - - return nil -} - -type SyncHarborPackage struct { - common.KubeAction -} - -func (g *SyncHarborPackage) Execute(runtime connector.Runtime) error { - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - binariesMapObj, ok := g.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) - if !ok { - return errors.New("get KubeBinary by pipeline cache failed") - } - binariesMap := binariesMapObj.(map[string]*files.KubeBinary) - - harbor, ok := binariesMap[common.Harbor] - if !ok { - return errors.New("get KubeBinary key harbor by pipeline cache failed") - } - - dst := filepath.Join(common.TmpDir, harbor.FileName) - if err := runtime.GetRunner().Scp(harbor.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), "sync harbor package failed") - } - - installCmd := fmt.Sprintf("tar -zxvf %s -C /opt", dst) - if _, err := runtime.GetRunner().SudoCmd(installCmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "unzip harbor package failed") - } - - return nil -} - -type GenerateHarborConfig struct { - common.KubeAction -} - -func (g *GenerateHarborConfig) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - templateAction := action.Template{ - Template: templates.HarborConfigTempl, - Dst: "/opt/harbor/harbor.yml", - Data: util.Data{ - "Domain": host.GetName(), - "Certificate": fmt.Sprintf("%s.pem", RegistryCertificateBaseName), - "Key": fmt.Sprintf("%s-key.pem", RegistryCertificateBaseName), - "Password": templates.Password(g.KubeConf, RegistryCertificateBaseName), - }, - } - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type StartHarbor struct { - common.KubeAction -} - -func (g *StartHarbor) Execute(runtime connector.Runtime) error { - startCmd := "cd /opt/harbor && chmod +x install.sh && export PATH=$PATH:/usr/local/bin; ./install.sh --with-notary --with-trivy --with-chartmuseum && systemctl daemon-reload && systemctl enable harbor && systemctl restart harbor" - if _, err := runtime.GetRunner().SudoCmd(startCmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "start harbor failed") - } - - fmt.Println() - fmt.Println(fmt.Sprintf("Local image registry created successfully. Address: %s", g.KubeConf.Cluster.Registry.PrivateRegistry)) - fmt.Println() - - return nil -} diff --git a/cmd/kk/pkg/bootstrap/registry/templates/harbor.go b/cmd/kk/pkg/bootstrap/registry/templates/harbor.go deleted file mode 100644 index 7e97a46d3..000000000 --- a/cmd/kk/pkg/bootstrap/registry/templates/harbor.go +++ /dev/null @@ -1,149 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package templates - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/registry" - "strings" - "text/template" - - "github.com/lithammer/dedent" -) - -var ( - // HarborServiceTempl defines the template of registry's configuration file. - HarborServiceTempl = template.Must(template.New("harborSerivce").Parse( - dedent.Dedent(`[Unit] -Description=Harbor -After=docker.service systemd-networkd.service systemd-resolved.service -Requires=docker.service - -[Service] -Type=simple -ExecStart=/usr/local/bin/docker-compose -f {{ .Harbor_install_path }}/docker-compose.yml up -ExecStop=/usr/local/bin/docker-compose -f {{ .Harbor_install_path }}/docker-compose.yml down -Restart=on-failure -[Install] -WantedBy=multi-user.target - `))) - // HarborConfigTempl defines the template of registry's configuration file. - HarborConfigTempl = template.Must(template.New("harborConfig").Parse( - dedent.Dedent(`# Configuration file of Harbor - -# The IP address or hostname to access admin UI and registry service. -# DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients. -hostname: {{ .Domain }} - -# http related config -http: - # port for http, default is 80. If https enabled, this port will redirect to https port - port: 80 - -# https related config -https: - # https port for harbor, default is 443 - port: 443 - # The path of cert and key files for nginx - certificate: /etc/ssl/registry/ssl/{{ .Certificate }} - private_key: /etc/ssl/registry/ssl/{{ .Key }} - -# The initial password of Harbor admin -# It only works in first time to install harbor -# Remember Change the admin password from UI after launching Harbor. -harbor_admin_password: Harbor12345 - -# Harbor DB configuration -database: - # The password for the root user of Harbor DB. Change this before any production use. - password: root123 - # The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained. - max_idle_conns: 100 - # The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections. - # Note: the default number of connections is 1024 for postgres of harbor. - max_open_conns: 900 - -# The default data volume -data_volume: /mnt/registry - -# Trivy configuration -# -# Trivy DB contains vulnerability information from NVD, Red Hat, and many other upstream vulnerability databases. -# It is downloaded by Trivy from the GitHub release page https://github.com/aquasecurity/trivy-db/releases and cached -# in the local file system. In addition, the database contains the update timestamp so Trivy can detect whether it -# should download a newer version from the Internet or use the cached one. Currently, the database is updated every -# 12 hours and published as a new release to GitHub. -trivy: - # ignoreUnfixed The flag to display only fixed vulnerabilities - ignore_unfixed: false - # skipUpdate The flag to enable or disable Trivy DB downloads from GitHub - # - skip_update: false - # - # insecure The flag to skip verifying registry certificate - insecure: false - -jobservice: - # Maximum number of job workers in job service - max_job_workers: 10 - -notification: - # Maximum retry count for webhook job - webhook_job_max_retry: 10 - -chart: - # Change the value of absolute_url to enabled can enable absolute url in chart - absolute_url: disabled - -# Log configurations -log: - # options are debug, info, warning, error, fatal - level: info - # configs for logs in local storage - local: - # Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated. - rotate_count: 50 - # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. - # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G - # are all valid. - rotate_size: 200M - # The directory on your host that store log - location: /var/log/harbor - -#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY! -_version: 2.4.0 - -# Global proxy -proxy: - http_proxy: - https_proxy: - no_proxy: - components: - - core - - jobservice - - trivy - - `))) -) - -func Password(kubeConf *common.KubeConf, domain string) string { - auths := registry.DockerRegistryAuthEntries(kubeConf.Cluster.Registry.Auths) - for repo, entry := range auths { - if strings.Contains(repo, domain) { - return entry.Password - } - } - - return "Harbor12345" -} \ No newline at end of file diff --git a/cmd/kk/pkg/bootstrap/registry/templates/registry.go b/cmd/kk/pkg/bootstrap/registry/templates/registry.go deleted file mode 100644 index 527935f19..000000000 --- a/cmd/kk/pkg/bootstrap/registry/templates/registry.go +++ /dev/null @@ -1,52 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var ( - // RegistryServiceTempl defines the template of registry service for systemd. - RegistryServiceTempl = template.Must(template.New("registryService").Parse( - dedent.Dedent(`[Unit] -Description=v2 Registry server for Container -After=network.target -[Service] -Type=simple -ExecStart=/usr/local/bin/registry serve /etc/kubekey/registry/config.yaml -Restart=on-failure -[Install] -WantedBy=multi-user.target - `))) - - // RegistryConfigTempl defines the template of registry's configuration file. - RegistryConfigTempl = template.Must(template.New("registryConfig").Parse( - dedent.Dedent(`version: 0.1 -log: - fields: - service: registry -storage: - cache: - layerinfo: inmemory - filesystem: - rootdirectory: /mnt/registry -http: - addr: :443 - tls: - certificate: /etc/ssl/registry/ssl/{{ .Certificate }} - key: /etc/ssl/registry/ssl/{{ .Key }} - `))) -) diff --git a/cmd/kk/pkg/certs/module.go b/cmd/kk/pkg/certs/module.go deleted file mode 100644 index 34130b9d4..000000000 --- a/cmd/kk/pkg/certs/module.go +++ /dev/null @@ -1,190 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package certs - -import ( - "path/filepath" - - versionutil "k8s.io/apimachinery/pkg/util/version" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/certs/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" -) - -type CheckCertsModule struct { - common.KubeModule -} - -func (c *CheckCertsModule) Init() { - c.Name = "CheckCertsModule" - c.Desc = "Check cluster certs" - - check := &task.RemoteTask{ - Name: "CheckClusterCerts", - Desc: "Check cluster certs", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Action: new(ListClusterCerts), - Parallel: true, - } - - c.Tasks = []task.Interface{ - check, - } -} - -type PrintClusterCertsModule struct { - common.KubeModule -} - -func (p *PrintClusterCertsModule) Init() { - p.Name = "PrintClusterCertsModule" - p.Desc = "Display cluster certs form" - - display := &task.LocalTask{ - Name: "DisplayCertsForm", - Desc: "Display cluster certs form", - Action: new(DisplayForm), - } - - p.Tasks = []task.Interface{ - display, - } -} - -type RenewCertsModule struct { - common.KubeModule -} - -func (r *RenewCertsModule) Init() { - r.Name = "RenewCertsModule" - r.Desc = "Renew control-plane certs" - - renew := &task.RemoteTask{ - Name: "RenewCerts", - Desc: "Renew control-plane certs", - Hosts: r.Runtime.GetHostsByRole(common.Master), - Action: new(RenewCerts), - Parallel: false, - Retry: 5, - } - - copyKubeConfig := &task.RemoteTask{ - Name: "CopyKubeConfig", - Desc: "Copy admin.conf to ~/.kube/config", - Hosts: r.Runtime.GetHostsByRole(common.Master), - Action: new(kubernetes.CopyKubeConfigForControlPlane), - Parallel: true, - Retry: 2, - } - - r.Tasks = []task.Interface{ - renew, - copyKubeConfig, - } -} - -type AutoRenewCertsModule struct { - common.KubeModule - Skip bool -} - -func (a *AutoRenewCertsModule) IsSkip() bool { - return a.Skip -} - -func (a *AutoRenewCertsModule) Init() { - a.Name = "AutoRenewCertsModule" - a.Desc = "Install auto renew control-plane certs" - - generateK8sCertsRenewScript := &task.RemoteTask{ - Name: "GenerateK8sCertsRenewScript", - Desc: "Generate k8s certs renew script", - Hosts: a.Runtime.GetHostsByRole(common.Master), - Action: &action.Template{ - Template: templates.K8sCertsRenewScript, - Dst: filepath.Join("/usr/local/bin/kube-scripts/", templates.K8sCertsRenewScript.Name()), - Data: util.Data{ - "IsDocker": a.KubeConf.Cluster.Kubernetes.ContainerManager == common.Docker, - "IsKubeadmAlphaCerts": versionutil.MustParseSemantic(a.KubeConf.Cluster.Kubernetes.Version).LessThan(versionutil.MustParseGeneric("v1.20.0")), - }, - }, - Parallel: true, - } - - generateK8sCertsRenewService := &task.RemoteTask{ - Name: "GenerateK8sCertsRenewService", - Desc: "Generate k8s certs renew service", - Hosts: a.Runtime.GetHostsByRole(common.Master), - Action: &action.Template{ - Template: templates.K8sCertsRenewService, - Dst: filepath.Join("/etc/systemd/system/", templates.K8sCertsRenewService.Name()), - }, - Parallel: true, - } - - generateK8sCertsRenewTimer := &task.RemoteTask{ - Name: "GenerateK8sCertsRenewTimer", - Desc: "Generate k8s certs renew timer", - Hosts: a.Runtime.GetHostsByRole(common.Master), - Action: &action.Template{ - Template: templates.K8sCertsRenewTimer, - Dst: filepath.Join("/etc/systemd/system/", templates.K8sCertsRenewTimer.Name()), - }, - Parallel: true, - } - - enable := &task.RemoteTask{ - Name: "EnableK8sCertsRenewService", - Desc: "Enable k8s certs renew service", - Hosts: a.Runtime.GetHostsByRole(common.Master), - Action: new(EnableRenewService), - Parallel: true, - } - - a.Tasks = []task.Interface{ - generateK8sCertsRenewScript, - generateK8sCertsRenewService, - generateK8sCertsRenewTimer, - enable, - } -} - -type UninstallAutoRenewCertsModule struct { - common.KubeModule -} - -func (u *UninstallAutoRenewCertsModule) Init() { - u.Name = "UninstallAutoRenewCertsModule" - u.Desc = "UnInstall auto renew control-plane certs" - - uninstall := &task.RemoteTask{ - Name: "UnInstallAutoRenewCerts", - Desc: "UnInstall auto renew control-plane certs", - Hosts: u.Runtime.GetHostsByRole(common.Master), - Prepare: new(AutoRenewCertsEnabled), - Action: new(UninstallAutoRenewCerts), - Parallel: true, - } - - u.Tasks = []task.Interface{ - uninstall, - } -} diff --git a/cmd/kk/pkg/certs/prepares.go b/cmd/kk/pkg/certs/prepares.go deleted file mode 100644 index ff8f69d62..000000000 --- a/cmd/kk/pkg/certs/prepares.go +++ /dev/null @@ -1,41 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package certs - -import ( - "path/filepath" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/certs/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type AutoRenewCertsEnabled struct { - common.KubePrepare - Not bool -} - -func (a *AutoRenewCertsEnabled) PreCheck(runtime connector.Runtime) (bool, error) { - exist, err := runtime.GetRunner().FileExist(filepath.Join("/etc/systemd/system/", templates.K8sCertsRenewService.Name())) - if err != nil { - return false, err - } - if exist { - return !a.Not, nil - } - return a.Not, nil -} diff --git a/cmd/kk/pkg/certs/tasks.go b/cmd/kk/pkg/certs/tasks.go deleted file mode 100644 index 831661463..000000000 --- a/cmd/kk/pkg/certs/tasks.go +++ /dev/null @@ -1,409 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package certs - -import ( - "encoding/base64" - "fmt" - "os" - "path/filepath" - "strings" - "text/tabwriter" - "time" - - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/runtime/schema" - versionutil "k8s.io/apimachinery/pkg/util/version" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - clientcmdlatest "k8s.io/client-go/tools/clientcmd/api/latest" - certutil "k8s.io/client-go/util/cert" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/certs/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" -) - -type Certificate struct { - Name string - Expires string - Residual string - AuthorityName string - NodeName string -} - -type CaCertificate struct { - AuthorityName string - Expires string - Residual string - NodeName string -} - -var ( - certificateList = []string{ - "apiserver.crt", - "apiserver-kubelet-client.crt", - "front-proxy-client.crt", - } - caCertificateList = []string{ - "ca.crt", - "front-proxy-ca.crt", - } - kubeConfigList = []string{ - "admin.conf", - "controller-manager.conf", - "scheduler.conf", - } -) - -type ListClusterCerts struct { - common.KubeAction -} - -func (l *ListClusterCerts) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - - certificates := make([]*Certificate, 0) - caCertificates := make([]*CaCertificate, 0) - - for _, certFileName := range certificateList { - certPath := filepath.Join(common.KubeCertDir, certFileName) - certContext, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("cat %s", certPath), false) - if err != nil { - return errors.Wrap(err, "get cluster certs failed") - } - if cert, err := getCertInfo(certContext, certFileName, host.GetName()); err != nil { - return err - } else { - certificates = append(certificates, cert) - } - } - for _, kubeConfigFileName := range kubeConfigList { - kubeConfigPath := filepath.Join(common.KubeConfigDir, kubeConfigFileName) - newConfig := clientcmdapi.NewConfig() - kubeconfigBytes, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("cat %s", kubeConfigPath), false) - decoded, _, err := clientcmdlatest.Codec.Decode([]byte(kubeconfigBytes), &schema.GroupVersionKind{Version: clientcmdlatest.Version, Kind: "Config"}, newConfig) - if err != nil { - return err - } - newConfig = decoded.(*clientcmdapi.Config) - for _, a := range newConfig.AuthInfos { - certContextBase64 := a.ClientCertificateData - tmp := base64.StdEncoding.EncodeToString(certContextBase64) - certContext, err := base64.StdEncoding.DecodeString(tmp) - if err != nil { - return err - } - if cert, err := getCertInfo(string(certContext), kubeConfigFileName, host.GetName()); err != nil { - return err - } else { - certificates = append(certificates, cert) - } - } - } - - for _, caCertFileName := range caCertificateList { - certPath := filepath.Join(common.KubeCertDir, caCertFileName) - caCertContext, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("cat %s", certPath), false) - if err != nil { - return errors.Wrap(err, "Failed to get cluster certs") - } - if cert, err := getCaCertInfo(caCertContext, caCertFileName, host.GetName()); err != nil { - return err - } else { - caCertificates = append(caCertificates, cert) - } - } - - host.GetCache().Set(common.Certificate, certificates) - host.GetCache().Set(common.CaCertificate, caCertificates) - return nil -} - -func getCertInfo(certContext, certFileName, nodeName string) (*Certificate, error) { - certs, err1 := certutil.ParseCertsPEM([]byte(certContext)) - if err1 != nil { - return nil, errors.Wrap(err1, "Failed to get cluster certs") - } - var authorityName string - switch certFileName { - case "apiserver.crt": - authorityName = "ca" - case "apiserver-kubelet-client.crt": - authorityName = "ca" - case "front-proxy-client.crt": - authorityName = "front-proxy-ca" - default: - authorityName = "" - } - cert := Certificate{ - Name: certFileName, - Expires: certs[0].NotAfter.Format("Jan 02, 2006 15:04 MST"), - Residual: ResidualTime(certs[0].NotAfter), - AuthorityName: authorityName, - NodeName: nodeName, - } - return &cert, nil -} - -func getCaCertInfo(certContext, certFileName, nodeName string) (*CaCertificate, error) { - certs, err := certutil.ParseCertsPEM([]byte(certContext)) - if err != nil { - return nil, errors.Wrap(err, "Failed to get cluster certs") - } - cert1 := CaCertificate{ - AuthorityName: certFileName, - Expires: certs[0].NotAfter.Format("Jan 02, 2006 15:04 MST"), - Residual: ResidualTime(certs[0].NotAfter), - NodeName: nodeName, - } - return &cert1, nil -} - -func ResidualTime(t time.Time) string { - d := time.Until(t) - if seconds := int(d.Seconds()); seconds < -1 { - return fmt.Sprintf("") - } else if seconds < 0 { - return fmt.Sprintf("0s") - } else if seconds < 60 { - return fmt.Sprintf("%ds", seconds) - } else if minutes := int(d.Minutes()); minutes < 60 { - return fmt.Sprintf("%dm", minutes) - } else if hours := int(d.Hours()); hours < 24 { - return fmt.Sprintf("%dh", hours) - } else if hours < 24*365 { - return fmt.Sprintf("%dd", hours/24) - } - return fmt.Sprintf("%dy", int(d.Hours()/24/365)) -} - -type DisplayForm struct { - common.KubeAction -} - -func (d *DisplayForm) Execute(runtime connector.Runtime) error { - certificates := make([]*Certificate, 0) - caCertificates := make([]*CaCertificate, 0) - - for _, host := range runtime.GetHostsByRole(common.Master) { - certs, ok := host.GetCache().Get(common.Certificate) - if !ok { - return errors.New("get certificate failed by pipeline cache") - } - ca, ok := host.GetCache().Get(common.CaCertificate) - if !ok { - return errors.New("get ca certificate failed by pipeline cache") - } - hostCertificates := certs.([]*Certificate) - hostCaCertificates := ca.([]*CaCertificate) - certificates = append(certificates, hostCertificates...) - caCertificates = append(caCertificates, hostCaCertificates...) - } - - w := tabwriter.NewWriter(os.Stdout, 10, 4, 3, ' ', 0) - _, _ = fmt.Fprintln(w, "CERTIFICATE\tEXPIRES\tRESIDUAL TIME\tCERTIFICATE AUTHORITY\tNODE") - for _, cert := range certificates { - s := fmt.Sprintf("%s\t%s\t%s\t%s\t%-8v", - cert.Name, - cert.Expires, - cert.Residual, - cert.AuthorityName, - cert.NodeName, - ) - - _, _ = fmt.Fprintln(w, s) - continue - } - _, _ = fmt.Fprintln(w) - _, _ = fmt.Fprintln(w, "CERTIFICATE AUTHORITY\tEXPIRES\tRESIDUAL TIME\tNODE") - for _, caCert := range caCertificates { - c := fmt.Sprintf("%s\t%s\t%s\t%-8v", - caCert.AuthorityName, - caCert.Expires, - caCert.Residual, - caCert.NodeName, - ) - - _, _ = fmt.Fprintln(w, c) - continue - } - - _ = w.Flush() - return nil -} - -type RenewCerts struct { - common.KubeAction -} - -func (r *RenewCerts) Execute(runtime connector.Runtime) error { - var kubeadmAlphaList = []string{ - "/usr/local/bin/kubeadm alpha certs renew apiserver", - "/usr/local/bin/kubeadm alpha certs renew apiserver-kubelet-client", - "/usr/local/bin/kubeadm alpha certs renew front-proxy-client", - "/usr/local/bin/kubeadm alpha certs renew admin.conf", - "/usr/local/bin/kubeadm alpha certs renew controller-manager.conf", - "/usr/local/bin/kubeadm alpha certs renew scheduler.conf", - } - - var kubeadmList = []string{ - "/usr/local/bin/kubeadm certs renew apiserver", - "/usr/local/bin/kubeadm certs renew apiserver-kubelet-client", - "/usr/local/bin/kubeadm certs renew front-proxy-client", - "/usr/local/bin/kubeadm certs renew admin.conf", - "/usr/local/bin/kubeadm certs renew controller-manager.conf", - "/usr/local/bin/kubeadm certs renew scheduler.conf", - } - - var restartList = []string{ - "docker ps -af name=k8s_kube-apiserver* -q | xargs --no-run-if-empty docker rm -f", - "docker ps -af name=k8s_kube-scheduler* -q | xargs --no-run-if-empty docker rm -f", - "docker ps -af name=k8s_kube-controller-manager* -q | xargs --no-run-if-empty docker rm -f", - "systemctl restart kubelet", - } - - version, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubeadm version -o short", true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "kubeadm get version failed") - } - cmp, err := versionutil.MustParseSemantic(version).Compare("v1.20.0") - if err != nil { - return errors.Wrap(errors.WithStack(err), "parse kubeadm version failed") - } - if cmp == -1 { - _, err := runtime.GetRunner().SudoCmd(strings.Join(kubeadmAlphaList, " && "), false) - if err != nil { - return errors.Wrap(err, "kubeadm alpha certs renew failed") - } - } else { - _, err := runtime.GetRunner().SudoCmd(strings.Join(kubeadmList, " && "), false) - if err != nil { - return errors.Wrap(err, "kubeadm alpha certs renew failed") - } - } - - _, err = runtime.GetRunner().SudoCmd(strings.Join(restartList, " && "), false) - if err != nil { - return errors.Wrap(err, "kube-apiserver, kube-schedule, kube-controller-manager or kubelet restart failed") - } - return nil -} - -type FetchKubeConfig struct { - common.KubeAction -} - -func (f *FetchKubeConfig) Execute(runtime connector.Runtime) error { - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - tmpConfigFile := filepath.Join(common.TmpDir, "admin.conf") - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("cp /etc/kubernetes/admin.conf %s", tmpConfigFile), false); err != nil { - return errors.Wrap(errors.WithStack(err), "copy kube config to /tmp/ failed") - } - - host := runtime.RemoteHost() - if err := runtime.GetRunner().Fetch(filepath.Join(runtime.GetWorkDir(), host.GetName(), "admin.conf"), tmpConfigFile); err != nil { - return errors.Wrap(errors.WithStack(err), "fetch kube config file failed") - } - return nil -} - -type SyneKubeConfigToWorker struct { - common.KubeAction -} - -func (s *SyneKubeConfigToWorker) Execute(runtime connector.Runtime) error { - createConfigDirCmd := "mkdir -p /root/.kube" - if _, err := runtime.GetRunner().SudoCmd(createConfigDirCmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "create .kube dir failed") - } - - firstMaster := runtime.GetHostsByRole(common.Master)[0] - localFile := filepath.Join(runtime.GetWorkDir(), firstMaster.GetName(), "admin.conf") - if err := runtime.GetRunner().SudoScp(localFile, "/root/.kube/config"); err != nil { - return errors.Wrap(errors.WithStack(err), "sudo scp config file to worker /root/.kube/config failed") - } - - // that doesn't work - //if err := runtime.GetRunner().SudoScp(filepath.Join(runtime.GetWorkDir(), firstMaster.GetName(), "admin.conf"), "$HOME/.kube/config"); err != nil { - // return errors.Wrap(errors.WithStack(err), "sudo scp config file to worker $HOME/.kube/config failed") - //} - - if host := runtime.RemoteHost(); host.GetUser() != "root" { - userConfigDirCmd := "mkdir -p $HOME/.kube" - if _, err := runtime.GetRunner().Cmd(userConfigDirCmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "user mkdir $HOME/.kube failed") - } - - getKubeConfigCmdUsr := "cp -f /root/.kube/config $HOME/.kube/config" - if _, err := runtime.GetRunner().SudoCmd(getKubeConfigCmdUsr, false); err != nil { - return errors.Wrap(errors.WithStack(err), "user copy /etc/kubernetes/admin.conf to $HOME/.kube/config failed") - } - - userId, err := runtime.GetRunner().Cmd("echo $(id -u)", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get user id failed") - } - - userGroupId, err := runtime.GetRunner().Cmd("echo $(id -g)", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get user group id failed") - } - - chownKubeConfig := fmt.Sprintf("chown -R %s:%s $HOME/.kube", userId, userGroupId) - if _, err := runtime.GetRunner().SudoCmd(chownKubeConfig, false); err != nil { - return errors.Wrap(errors.WithStack(err), "chown user kube config failed") - } - } - return nil -} - -type EnableRenewService struct { - common.KubeAction -} - -func (e *EnableRenewService) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd( - "chmod +x /usr/local/bin/kube-scripts/k8s-certs-renew.sh && systemctl enable --now k8s-certs-renew.timer", - false); err != nil { - return errors.Wrap(errors.WithStack(err), "enable k8s renew certs service failed") - } - return nil -} - -type UninstallAutoRenewCerts struct { - common.KubeAction -} - -func (u *UninstallAutoRenewCerts) Execute(runtime connector.Runtime) error { - _, _ = runtime.GetRunner().SudoCmd("systemctl disable k8s-certs-renew.timer 1>/dev/null 2>/dev/null", true) - _, _ = runtime.GetRunner().SudoCmd("systemctl stop k8s-certs-renew.timer 1>/dev/null 2>/dev/null", true) - - files := []string{ - filepath.Join("/usr/local/bin/kube-scripts/", templates.K8sCertsRenewScript.Name()), - filepath.Join("/etc/systemd/system/", templates.K8sCertsRenewService.Name()), - filepath.Join("/etc/systemd/system/", templates.K8sCertsRenewTimer.Name()), - } - for _, file := range files { - _, _ = runtime.GetRunner().SudoCmd(fmt.Sprintf("rm -rf %s", file), true) - } - - return nil -} diff --git a/cmd/kk/pkg/certs/templates/certs_renew_script.go b/cmd/kk/pkg/certs/templates/certs_renew_script.go deleted file mode 100644 index e7b590a20..000000000 --- a/cmd/kk/pkg/certs/templates/certs_renew_script.go +++ /dev/null @@ -1,55 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -// K8sCertsRenewScript defines the template of k8s-certs-renew timer for systemd. -var K8sCertsRenewScript = template.Must(template.New("k8s-certs-renew.sh").Parse( - dedent.Dedent(`#!/bin/bash -{{- if .IsKubeadmAlphaCerts }} -kubeadmCerts='/usr/local/bin/kubeadm alpha certs' -{{- else}} -kubeadmCerts='/usr/local/bin/kubeadm certs' -{{- end }} -getCertValidDays() { - local earliestExpireDate; earliestExpireDate=$(${kubeadmCerts} check-expiration | grep -o "[A-Za-z]\{3,4\}\s\w\w,\s[0-9]\{4,\}\s\w*:\w*\s\w*\s*" | xargs -I {} date -d {} +%s | sort | head -n 1) - local today; today="$(date +%s)" - echo -n $(( ($earliestExpireDate - $today) / (24 * 60 * 60) )) -} -echo "## Expiration before renewal ##" -${kubeadmCerts} check-expiration -if [ $(getCertValidDays) -lt 30 ]; then - echo "## Renewing certificates managed by kubeadm ##" - ${kubeadmCerts} renew all - echo "## Restarting control plane pods managed by kubeadm ##" -{{- if .IsDocker}} - $(which docker | grep docker) ps -af 'name=k8s_POD_(kube-apiserver|kube-controller-manager|kube-scheduler|etcd)-*' -q | /usr/bin/xargs $(which docker | grep docker) rm -f -{{- else}} - $(which crictl | grep crictl) pods --namespace kube-system --name 'kube-scheduler-*|kube-controller-manager-*|kube-apiserver-*|etcd-*' -q | /usr/bin/xargs $(which crictl | grep crictl) rmp -f -{{- end }} - echo "## Updating /root/.kube/config ##" - cp /etc/kubernetes/admin.conf /root/.kube/config -fi -echo "## Waiting for apiserver to be up again ##" -until printf "" 2>>/dev/null >>/dev/tcp/127.0.0.1/6443; do sleep 1; done -echo "## Expiration after renewal ##" -${kubeadmCerts} check-expiration - `))) diff --git a/cmd/kk/pkg/certs/templates/certs_renew_service.go b/cmd/kk/pkg/certs/templates/certs_renew_service.go deleted file mode 100644 index b29aa39f0..000000000 --- a/cmd/kk/pkg/certs/templates/certs_renew_service.go +++ /dev/null @@ -1,44 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var ( - // K8sCertsRenewService defines the template of k8s-certs-renew service for systemd. - K8sCertsRenewService = template.Must(template.New("k8s-certs-renew.service").Parse( - dedent.Dedent(`[Unit] -Description=Renew K8S control plane certificates -[Service] -Type=oneshot -ExecStart=/usr/local/bin/kube-scripts/k8s-certs-renew.sh - `))) - - // K8sCertsRenewTimer defines the template of k8s-certs-renew timer for systemd. - K8sCertsRenewTimer = template.Must(template.New("k8s-certs-renew.timer").Parse( - dedent.Dedent(`[Unit] -Description=Timer to renew K8S control plane certificates -[Timer] -OnCalendar=Mon *-*-* 03:00:00 -Unit=k8s-certs-renew.service -[Install] -WantedBy=multi-user.target - `))) -) diff --git a/cmd/kk/pkg/client/kubernetes/client.go b/cmd/kk/pkg/client/kubernetes/client.go deleted file mode 100644 index f40a9463f..000000000 --- a/cmd/kk/pkg/client/kubernetes/client.go +++ /dev/null @@ -1,43 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubernetes - -import ( - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - "os" -) - -func NewClient(kubeConfig string) (*kubernetes.Clientset, error) { - data, err := os.ReadFile(kubeConfig) - if err != nil { - return nil, err - } - config, err := clientcmd.NewClientConfigFromBytes(data) - if err != nil { - return nil, err - } - restConfig, err := config.ClientConfig() - if err != nil { - return nil, err - } - client, err := kubernetes.NewForConfig(restConfig) - if err != nil { - return nil, err - } - return client, nil -} diff --git a/cmd/kk/pkg/common/artifact_action.go b/cmd/kk/pkg/common/artifact_action.go deleted file mode 100644 index c2f60c0c7..000000000 --- a/cmd/kk/pkg/common/artifact_action.go +++ /dev/null @@ -1,37 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type ArtifactAction struct { - action.BaseAction - Manifest *ArtifactManifest -} - -func (a *ArtifactAction) AutoAssert(runtime connector.Runtime) { - artifactRuntime := runtime.(*ArtifactRuntime) - m := &ArtifactManifest{ - Spec: artifactRuntime.Spec, - Arg: artifactRuntime.Arg, - } - - a.Manifest = m -} diff --git a/cmd/kk/pkg/common/artifact_module.go b/cmd/kk/pkg/common/artifact_module.go deleted file mode 100644 index aa850f0c5..000000000 --- a/cmd/kk/pkg/common/artifact_module.go +++ /dev/null @@ -1,46 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -import ( - kubekeyv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" -) - -type ArtifactManifest struct { - Spec *kubekeyv1alpha2.ManifestSpec - Arg ArtifactArgument -} - -type ArtifactModule struct { - module.BaseTaskModule - Manifest *ArtifactManifest -} - -func (a *ArtifactModule) IsSkip() bool { - return a.Skip -} - -func (a *ArtifactModule) AutoAssert() { - artifactRuntime := a.Runtime.(*ArtifactRuntime) - m := &ArtifactManifest{ - Spec: artifactRuntime.Spec, - Arg: artifactRuntime.Arg, - } - - a.Manifest = m -} diff --git a/cmd/kk/pkg/common/artifact_prepare.go b/cmd/kk/pkg/common/artifact_prepare.go deleted file mode 100644 index 92423e29f..000000000 --- a/cmd/kk/pkg/common/artifact_prepare.go +++ /dev/null @@ -1,37 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" -) - -type ArtifactPrepare struct { - prepare.BasePrepare - Manifest *ArtifactManifest -} - -func (a *ArtifactPrepare) AutoAssert(runtime connector.Runtime) { - artifactRuntime := runtime.(*ArtifactRuntime) - m := &ArtifactManifest{ - Spec: artifactRuntime.Spec, - Arg: artifactRuntime.Arg, - } - - a.Manifest = m -} diff --git a/cmd/kk/pkg/common/artifact_runtime.go b/cmd/kk/pkg/common/artifact_runtime.go deleted file mode 100644 index fdd453bf7..000000000 --- a/cmd/kk/pkg/common/artifact_runtime.go +++ /dev/null @@ -1,84 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -import ( - "encoding/json" - "os" - "path/filepath" - - "github.com/pkg/errors" - k8syaml "k8s.io/apimachinery/pkg/util/yaml" - - kubekeyv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type ArtifactArgument struct { - ManifestFile string - Output string - CriSocket string - Debug bool - IgnoreErr bool - DownloadCommand func(path, url string) string -} - -type ArtifactRuntime struct { - LocalRuntime - Spec *kubekeyv1alpha2.ManifestSpec - Arg ArtifactArgument -} - -func NewArtifactRuntime(arg ArtifactArgument) (*ArtifactRuntime, error) { - localRuntime, err := NewLocalRuntime(arg.Debug, arg.IgnoreErr) - if err != nil { - return nil, err - } - - fp, err := filepath.Abs(arg.ManifestFile) - if err != nil { - return nil, errors.Wrap(err, "Failed to look up current directory") - } - - fileByte, err := os.ReadFile(fp) - if err != nil { - return nil, errors.Wrapf(err, "Failed to read file %s", fp) - } - - contentToJson, err := k8syaml.ToJSON(fileByte) - if err != nil { - return nil, errors.Wrap(err, "Unable to convert configuration to json") - } - - manifest := &kubekeyv1alpha2.Manifest{} - if err := json.Unmarshal(contentToJson, manifest); err != nil { - return nil, errors.Wrapf(err, "Failed to json unmarshal") - } - - r := &ArtifactRuntime{ - Spec: &manifest.Spec, - Arg: arg, - } - r.LocalRuntime = localRuntime - return r, nil -} - -// Copy is used to create a copy for Runtime. -func (a *ArtifactRuntime) Copy() connector.Runtime { - runtime := *a - return &runtime -} diff --git a/cmd/kk/pkg/common/common.go b/cmd/kk/pkg/common/common.go deleted file mode 100644 index 4b24d7a3e..000000000 --- a/cmd/kk/pkg/common/common.go +++ /dev/null @@ -1,98 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -const ( - K3s = "k3s" - K8e = "k8e" - Kubernetes = "kubernetes" - - LocalHost = "localhost" - - AllInOne = "allInOne" - File = "file" - Operator = "operator" - - Master = "master" - Worker = "worker" - ETCD = "etcd" - K8s = "k8s" - Registry = "registry" - KubeKey = "kubekey" - Harbor = "harbor" - DockerCompose = "compose" - - KubeBinaries = "KubeBinaries" - - TmpDir = "/tmp/kubekey" - BinDir = "/usr/local/bin" - KubeConfigDir = "/etc/kubernetes" - KubeAddonsDir = "/etc/kubernetes/addons" - KubeCertDir = "/etc/kubernetes/pki" - KubeManifestDir = "/etc/kubernetes/manifests" - KubeScriptDir = "/usr/local/bin/kube-scripts" - KubeletFlexvolumesPluginsDir = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec" - - ETCDCertDir = "/etc/ssl/etcd/ssl" - RegistryCertDir = "/etc/ssl/registry/ssl" - - HaproxyDir = "/etc/kubekey/haproxy" - - IPv4Regexp = "[\\d]+\\.[\\d]+\\.[\\d]+\\.[\\d]+" - IPv6Regexp = "[a-f0-9]{1,4}(:[a-f0-9]{1,4}){7}|[a-f0-9]{1,4}(:[a-f0-9]{1,4}){0,7}::[a-f0-9]{0,4}(:[a-f0-9]{1,4}){0,7}" - - Calico = "calico" - Flannel = "flannel" - Cilium = "cilium" - Kubeovn = "kubeovn" - Hybridnet = "hybridnet" - - Docker = "docker" - Crictl = "crictl" - Containerd = "containerd" - Crio = "crio" - Isula = "isula" - Runc = "runc" - - // global cache key - // PreCheckModule - NodePreCheck = "nodePreCheck" - K8sVersion = "k8sVersion" // current k8s version - MaxK8sVersion = "maxK8sVersion" // max k8s version of nodes - KubeSphereVersion = "kubeSphereVersion" // current KubeSphere version - ClusterNodeStatus = "clusterNodeStatus" - ClusterNodeCRIRuntimes = "ClusterNodeCRIRuntimes" - DesiredK8sVersion = "desiredK8sVersion" - PlanK8sVersion = "planK8sVersion" - NodeK8sVersion = "NodeK8sVersion" - - // ETCDModule - ETCDCluster = "etcdCluster" - ETCDName = "etcdName" - ETCDExist = "etcdExist" - - // KubernetesModule - ClusterStatus = "clusterStatus" - ClusterExist = "clusterExist" - - // CertsModule - Certificate = "certificate" - CaCertificate = "caCertificate" - - // Artifact pipeline - Artifact = "artifact" -) diff --git a/cmd/kk/pkg/common/kube_action.go b/cmd/kk/pkg/common/kube_action.go deleted file mode 100644 index 346d4dc97..000000000 --- a/cmd/kk/pkg/common/kube_action.go +++ /dev/null @@ -1,39 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type KubeAction struct { - action.BaseAction - KubeConf *KubeConf -} - -func (k *KubeAction) AutoAssert(runtime connector.Runtime) { - kubeRuntime := runtime.(*KubeRuntime) - conf := &KubeConf{ - Cluster: kubeRuntime.Cluster, - ClusterName: kubeRuntime.ClusterName, - Kubeconfig: kubeRuntime.Kubeconfig, - Arg: kubeRuntime.Arg, - } - - k.KubeConf = conf -} diff --git a/cmd/kk/pkg/common/kube_module.go b/cmd/kk/pkg/common/kube_module.go deleted file mode 100644 index 38f4191a6..000000000 --- a/cmd/kk/pkg/common/kube_module.go +++ /dev/null @@ -1,68 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -import ( - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" -) - -type KubeConf struct { - ClusterHosts []string - ClusterName string - Cluster *kubekeyapiv1alpha2.ClusterSpec - Kubeconfig string - Arg Argument -} - -type KubeModule struct { - module.BaseTaskModule - KubeConf *KubeConf -} - -func (k *KubeModule) IsSkip() bool { - return k.Skip -} - -func (k *KubeModule) AutoAssert() { - kubeRuntime := k.Runtime.(*KubeRuntime) - conf := &KubeConf{ - ClusterName: kubeRuntime.ClusterName, - Cluster: kubeRuntime.Cluster, - Kubeconfig: kubeRuntime.Kubeconfig, - Arg: kubeRuntime.Arg, - } - - k.KubeConf = conf -} - -type KubeCustomModule struct { - module.CustomModule - KubeConf *KubeConf -} - -func (k *KubeCustomModule) AutoAssert() { - kubeRuntime := k.Runtime.(*KubeRuntime) - conf := &KubeConf{ - ClusterName: kubeRuntime.ClusterName, - Cluster: kubeRuntime.Cluster, - Kubeconfig: kubeRuntime.Kubeconfig, - Arg: kubeRuntime.Arg, - } - - k.KubeConf = conf -} diff --git a/cmd/kk/pkg/common/kube_prepare.go b/cmd/kk/pkg/common/kube_prepare.go deleted file mode 100644 index 0ee6b9835..000000000 --- a/cmd/kk/pkg/common/kube_prepare.go +++ /dev/null @@ -1,138 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" -) - -type KubePrepare struct { - prepare.BasePrepare - KubeConf *KubeConf -} - -func (k *KubePrepare) AutoAssert(runtime connector.Runtime) { - kubeRuntime := runtime.(*KubeRuntime) - conf := &KubeConf{ - Cluster: kubeRuntime.Cluster, - Kubeconfig: kubeRuntime.Kubeconfig, - Arg: kubeRuntime.Arg, - } - - k.KubeConf = conf -} - -type OnlyFirstMaster struct { - KubePrepare - Not bool -} - -func (o *OnlyFirstMaster) PreCheck(runtime connector.Runtime) (bool, error) { - if runtime.RemoteHost().IsRole(Master) && - runtime.RemoteHost().GetName() == runtime.GetHostsByRole(Master)[0].GetName() { - return !o.Not, nil - } - return o.Not, nil -} - -type IsMaster struct { - KubePrepare -} - -func (i *IsMaster) PreCheck(runtime connector.Runtime) (bool, error) { - if runtime.RemoteHost().IsRole(Master) { - return true, nil - } - return false, nil -} - -type IsWorker struct { - KubePrepare - Not bool -} - -func (i *IsWorker) PreCheck(runtime connector.Runtime) (bool, error) { - if runtime.RemoteHost().IsRole(Worker) { - return !i.Not, nil - } - return i.Not, nil -} - -type OnlyWorker struct { - KubePrepare -} - -func (o *OnlyWorker) PreCheck(runtime connector.Runtime) (bool, error) { - if runtime.RemoteHost().IsRole(Worker) && !runtime.RemoteHost().IsRole(Master) { - return true, nil - } - return false, nil -} - -type OnlyETCD struct { - KubePrepare - Not bool -} - -func (o *OnlyETCD) PreCheck(runtime connector.Runtime) (bool, error) { - if runtime.RemoteHost().IsRole(ETCD) { - return !o.Not, nil - } - return o.Not, nil -} - -type OnlyK3s struct { - KubePrepare -} - -func (o *OnlyK3s) PreCheck(_ connector.Runtime) (bool, error) { - if o.KubeConf.Cluster.Kubernetes.Type == "k3s" { - return true, nil - } - return false, nil -} - -type OnlyKubernetes struct { - KubePrepare -} - -func (o *OnlyKubernetes) PreCheck(_ connector.Runtime) (bool, error) { - if o.KubeConf.Cluster.Kubernetes.Type != "k3s" { - return true, nil - } - return false, nil -} - -type EnableKubeProxy struct { - KubePrepare -} - -func (e *EnableKubeProxy) PreCheck(_ connector.Runtime) (bool, error) { - if !e.KubeConf.Cluster.Kubernetes.DisableKubeProxy { - return true, nil - } - return false, nil -} - -type EnableAudit struct { - KubePrepare -} - -func (e *EnableAudit) PreCheck(_ connector.Runtime) (bool, error) { - return e.KubeConf.Cluster.Kubernetes.EnableAudit(), nil -} diff --git a/cmd/kk/pkg/common/kube_rollback.go b/cmd/kk/pkg/common/kube_rollback.go deleted file mode 100644 index 00dee4c54..000000000 --- a/cmd/kk/pkg/common/kube_rollback.go +++ /dev/null @@ -1,39 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/rollback" -) - -type KubeRollback struct { - rollback.BaseRollback - KubeConf *KubeConf -} - -func (k *KubeRollback) AutoAssert(runtime connector.Runtime) { - kubeRuntime := runtime.(*KubeRuntime) - conf := &KubeConf{ - Cluster: kubeRuntime.Cluster, - ClusterName: kubeRuntime.ClusterName, - Kubeconfig: kubeRuntime.Kubeconfig, - Arg: kubeRuntime.Arg, - } - - k.KubeConf = conf -} diff --git a/cmd/kk/pkg/common/kube_runtime.go b/cmd/kk/pkg/common/kube_runtime.go deleted file mode 100644 index 970e924e9..000000000 --- a/cmd/kk/pkg/common/kube_runtime.go +++ /dev/null @@ -1,100 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -import ( - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type KubeRuntime struct { - connector.BaseRuntime - ClusterName string - Cluster *kubekeyapiv1alpha2.ClusterSpec - Kubeconfig string - Arg Argument -} - -type Argument struct { - NodeName string - FilePath string - KubernetesVersion string - KsEnable bool - KsVersion string - Debug bool - IgnoreErr bool - SkipPullImages bool - SkipPushImages bool - SecurityEnhancement bool - DeployLocalStorage *bool - DownloadCommand func(path, url string) string - SkipConfirmCheck bool - ContainerManager string - FromCluster bool - KubeConfig string - Artifact string - InstallPackages bool - ImagesDir string - Namespace string - DeleteCRI bool - Role string - Type string -} - -func NewKubeRuntime(flag string, arg Argument) (*KubeRuntime, error) { - loader := NewLoader(flag, arg) - cluster, err := loader.Load() - if err != nil { - return nil, err - } - - base := connector.NewBaseRuntime(cluster.Name, connector.NewDialer(), arg.Debug, arg.IgnoreErr) - - clusterSpec := &cluster.Spec - defaultCluster, roleGroups := clusterSpec.SetDefaultClusterSpec() - - hostSet := make(map[string]struct{}) - for _, role := range roleGroups { - for _, host := range role { - if host.IsRole(Master) || host.IsRole(Worker) { - host.SetRole(K8s) - } - if _, ok := hostSet[host.GetName()]; !ok { - hostSet[host.GetName()] = struct{}{} - base.AppendHost(host) - base.AppendRoleMap(host) - } - } - } - - arg.KsEnable = defaultCluster.KubeSphere.Enabled - arg.KsVersion = defaultCluster.KubeSphere.Version - r := &KubeRuntime{ - Cluster: defaultCluster, - ClusterName: cluster.Name, - Arg: arg, - } - r.BaseRuntime = base - - return r, nil -} - -// Copy is used to create a copy for Runtime. -func (k *KubeRuntime) Copy() connector.Runtime { - runtime := *k - return &runtime -} diff --git a/cmd/kk/pkg/common/loader.go b/cmd/kk/pkg/common/loader.go deleted file mode 100644 index 06098c03c..000000000 --- a/cmd/kk/pkg/common/loader.go +++ /dev/null @@ -1,308 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -import ( - "bufio" - "encoding/json" - "fmt" - "os" - "os/exec" - "os/user" - "path/filepath" - "regexp" - "runtime" - "strings" - "time" - - "github.com/pkg/errors" - "gopkg.in/yaml.v3" - k8syaml "k8s.io/apimachinery/pkg/util/yaml" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere" -) - -var ( - kubeReleaseRegex = regexp.MustCompile(`^v?(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)([-0-9a-zA-Z_\.+]*)?$`) -) - -type Loader interface { - Load() (*kubekeyapiv1alpha2.Cluster, error) -} - -type Options map[string]interface{} - -func NewLoader(flag string, arg Argument) Loader { - switch flag { - case File: - return NewFileLoader(arg) - case Operator: - return &ConfigMapLoader{} - case AllInOne: - return NewDefaultLoader(arg) - default: - return NewDefaultLoader(arg) - } -} - -type DefaultLoader struct { - arg Argument - KubernetesVersion string - KubeSphereVersion string - KubeSphereEnable bool -} - -func NewDefaultLoader(arg Argument) *DefaultLoader { - return &DefaultLoader{ - arg: arg, - KubernetesVersion: arg.KubernetesVersion, - KubeSphereVersion: arg.KsVersion, - KubeSphereEnable: arg.KsEnable, - } -} - -func (d *DefaultLoader) Load() (*kubekeyapiv1alpha2.Cluster, error) { - u, _ := user.Current() - if u.Username != "root" { - return nil, errors.New(fmt.Sprintf("Current user is %s. Please use root!", u.Username)) - } - - allInOne := kubekeyapiv1alpha2.Cluster{} - if output, err := exec.Command("/bin/sh", "-c", "if [ ! -f \"$HOME/.ssh/id_rsa\" ]; then ssh-keygen -t rsa-sha2-512 -P \"\" -f $HOME/.ssh/id_rsa && ls $HOME/.ssh;fi;").CombinedOutput(); err != nil { - return nil, errors.New(fmt.Sprintf("Failed to generate public key: %v\n%s", err, string(output))) - } - if output, err := exec.Command("/bin/sh", "-c", "echo \"\n$(cat $HOME/.ssh/id_rsa.pub)\" >> $HOME/.ssh/authorized_keys && awk ' !x[$0]++{print > \"'$HOME'/.ssh/authorized_keys\"}' $HOME/.ssh/authorized_keys").CombinedOutput(); err != nil { - return nil, errors.New(fmt.Sprintf("Failed to copy public key to authorized_keys: %v\n%s", err, string(output))) - } - - hostname, err := os.Hostname() - if err != nil { - return nil, errors.New(fmt.Sprintf("Failed to get hostname: %v\n", err)) - } - - allInOne.Spec.Hosts = append(allInOne.Spec.Hosts, kubekeyapiv1alpha2.HostCfg{ - Name: hostname, - Address: util.LocalIP(), - InternalAddress: util.LocalIP(), - Port: kubekeyapiv1alpha2.DefaultSSHPort, - User: u.Name, - Password: "", - PrivateKeyPath: fmt.Sprintf("%s/.ssh/id_rsa", u.HomeDir), - Arch: runtime.GOARCH, - }) - - allInOne.Spec.RoleGroups = map[string][]string{ - Master: {hostname}, - ETCD: {hostname}, - Worker: {hostname}, - Registry: {hostname}, - } - allInOne.Spec.ControlPlaneEndpoint.Address = "127.0.0.1" - if ver := normalizedBuildVersion(d.KubernetesVersion); ver != "" { - s := strings.Split(ver, "-") - if len(s) > 1 { - allInOne.Spec.Kubernetes = kubekeyapiv1alpha2.Kubernetes{ - Version: s[0], - Type: s[1], - } - } else { - allInOne.Spec.Kubernetes = kubekeyapiv1alpha2.Kubernetes{ - Version: ver, - } - } - } else { - allInOne.Spec.Kubernetes = kubekeyapiv1alpha2.Kubernetes{ - Version: kubekeyapiv1alpha2.DefaultKubeVersion, - } - } - - if d.KubeSphereEnable { - ver := normalizedBuildVersion(d.KubeSphereVersion) - if ver == "" { - return nil, errors.New(fmt.Sprintf("Unsupported Kubesphere Version: %v\n", d.KubeSphereVersion)) - } - if err := defaultKSConfig(&allInOne.Spec.KubeSphere, ver); err != nil { - return nil, err - } - } - - if d.arg.ContainerManager != "" && d.arg.ContainerManager != Docker { - allInOne.Spec.Kubernetes.ContainerManager = d.arg.ContainerManager - } - - enableAutoRenewCerts := true - allInOne.Spec.Kubernetes.AutoRenewCerts = &enableAutoRenewCerts - - // must be a lower case - allInOne.Name = "kubekey" + time.Now().Format("2006-01-02") - return &allInOne, nil -} - -type FileLoader struct { - arg Argument - FilePath string - KubernetesVersion string - KubeSphereVersion string - KubeSphereEnable bool -} - -func NewFileLoader(arg Argument) *FileLoader { - return &FileLoader{ - arg: arg, - FilePath: arg.FilePath, - KubernetesVersion: arg.KubernetesVersion, - KubeSphereVersion: arg.KsVersion, - KubeSphereEnable: arg.KsEnable, - } -} - -func (f FileLoader) Load() (*kubekeyapiv1alpha2.Cluster, error) { - var objName string - - clusterCfg := kubekeyapiv1alpha2.Cluster{} - fp, err := filepath.Abs(f.FilePath) - if err != nil { - return nil, errors.Wrap(err, "Failed to look up current directory") - } - // fixme: It will lead to nil pointer err - //if len(f.KubernetesVersion) != 0 { - // _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("sed -i \"/version/s/\\:.*/\\: %s/g\" %s", f.KubernetesVersion, fp)).Run() - //} - file, err := os.Open(fp) - if err != nil { - return nil, errors.Wrap(err, "Unable to open the given cluster configuration file") - } - defer file.Close() - b1 := bufio.NewReader(file) - for { - result := make(map[string]interface{}) - content, err := k8syaml.NewYAMLReader(b1).Read() - if len(content) == 0 { - break - } - if err != nil { - return nil, errors.Wrap(err, "Unable to read the given cluster configuration file") - } - err = yaml.Unmarshal(content, &result) - if err != nil { - return nil, errors.Wrap(err, "Unable to unmarshal the given cluster configuration file") - } - - if result["kind"] == "Cluster" { - contentToJson, err := k8syaml.ToJSON(content) - if err != nil { - return nil, errors.Wrap(err, "Unable to convert configuration to json") - } - if err := json.Unmarshal(contentToJson, &clusterCfg); err != nil { - return nil, errors.Wrap(err, "Failed to unmarshal configuration") - } - metadata := result["metadata"].(map[string]interface{}) - objName = metadata["name"].(string) - } - - if result["kind"] == "ConfigMap" || result["kind"] == "ClusterConfiguration" { - metadata := result["metadata"].(map[string]interface{}) - labels := metadata["labels"].(map[string]interface{}) - clusterCfg.Spec.KubeSphere.Enabled = true - - v, ok := labels["version"] - if !ok { - return nil, errors.New("Unknown version") - } - - version := v.(string) - _, stable := kubesphere.StabledVersionSupport(version) - _, latest := kubesphere.LatestRelease(version) - _, dev := kubesphere.DevRelease(version) - if stable || latest || dev { - clusterCfg.Spec.KubeSphere.Configurations = "---\n" + string(content) - clusterCfg.Spec.KubeSphere.Version = version - } else { - return nil, errors.New(fmt.Sprintf("Unsupported KubeSphere version: %s", version)) - } - } - } - - if f.KubeSphereEnable { - ver := normalizedBuildVersion(f.KubeSphereVersion) - if ver == "" { - return nil, errors.New(fmt.Sprintf("Unsupported Kubesphere Version: %v\n", f.KubeSphereVersion)) - } - if err := defaultKSConfig(&clusterCfg.Spec.KubeSphere, ver); err != nil { - return nil, err - } - } - - if ver := normalizedBuildVersion(f.KubernetesVersion); ver != "" { - s := strings.Split(ver, "-") - if len(s) > 1 { - clusterCfg.Spec.Kubernetes.Version = s[0] - clusterCfg.Spec.Kubernetes.Type = s[1] - } else { - clusterCfg.Spec.Kubernetes.Version = ver - } - } - - if f.arg.ContainerManager != "" && f.arg.ContainerManager != Docker { - clusterCfg.Spec.Kubernetes.ContainerManager = f.arg.ContainerManager - } - - clusterCfg.Spec.Kubernetes.Version = normalizedBuildVersion(clusterCfg.Spec.Kubernetes.Version) - clusterCfg.Spec.KubeSphere.Version = normalizedBuildVersion(clusterCfg.Spec.KubeSphere.Version) - clusterCfg.Name = objName - return &clusterCfg, nil -} - -type ConfigMapLoader struct { -} - -func (c ConfigMapLoader) Load() (*kubekeyapiv1alpha2.Cluster, error) { - return nil, nil -} - -func defaultKSConfig(ks *kubekeyapiv1alpha2.KubeSphere, version string) error { - ks.Enabled = true - version = strings.TrimSpace(version) - ksInstaller, ok := kubesphere.StabledVersionSupport(version) - if ok { - ks.Version = ksInstaller.Version - ks.Configurations = ksInstaller.CCToString() - } else if latest, ok := kubesphere.LatestRelease(version); ok { - ks.Version = version - ks.Configurations = latest.CCToString() - } else if dev, ok := kubesphere.DevRelease(version); ok { - ks.Version = version - ks.Configurations = dev.CCToString() - } else { - return errors.New(fmt.Sprintf("Unsupported KubeSphere version: %s", version)) - } - return nil -} - -// normalizedBuildVersion used to returns normalized build version (with "v" prefix if needed) -// If input doesn't match known version pattern, returns empty string. -func normalizedBuildVersion(version string) string { - if kubeReleaseRegex.MatchString(version) { - if strings.HasPrefix(version, "v") { - return version - } - return "v" + version - } - return "" -} diff --git a/cmd/kk/pkg/common/local_runtime.go b/cmd/kk/pkg/common/local_runtime.go deleted file mode 100644 index 7902e7f49..000000000 --- a/cmd/kk/pkg/common/local_runtime.go +++ /dev/null @@ -1,75 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -import ( - "fmt" - "os" - "os/exec" - "os/user" - "runtime" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -type LocalRuntime struct { - connector.BaseRuntime -} - -func NewLocalRuntime(debug, ingoreErr bool) (LocalRuntime, error) { - var localRuntime LocalRuntime - u, err := user.Current() - if err != nil { - return localRuntime, err - } - if u.Username != "root" { - return localRuntime, fmt.Errorf("current user is %s. Please use root", u.Username) - } - - if output, err := exec.Command("/bin/sh", "-c", "if [ ! -f \"$HOME/.ssh/id_rsa\" ]; then ssh-keygen -t rsa-sha2-512 -P \"\" -f $HOME/.ssh/id_rsa && ls $HOME/.ssh;fi;").CombinedOutput(); err != nil { - return localRuntime, errors.New(fmt.Sprintf("Failed to generate public key: %v\n%s", err, string(output))) - } - if output, err := exec.Command("/bin/sh", "-c", "echo \"\n$(cat $HOME/.ssh/id_rsa.pub)\" >> $HOME/.ssh/authorized_keys && awk ' !x[$0]++{print > \"'$HOME'/.ssh/authorized_keys\"}' $HOME/.ssh/authorized_keys").CombinedOutput(); err != nil { - return localRuntime, errors.New(fmt.Sprintf("Failed to copy public key to authorized_keys: %v\n%s", err, string(output))) - } - - name, err := os.Hostname() - if err != nil { - return localRuntime, err - } - base := connector.NewBaseRuntime(name, connector.NewDialer(), debug, ingoreErr) - - host := connector.NewHost() - host.Name = name - host.Address = util.LocalIP() - host.InternalAddress = util.LocalIP() - host.Port = 22 - host.User = u.Name - host.Password = "" - host.PrivateKeyPath = fmt.Sprintf("%s/.ssh/id_rsa", u.HomeDir) - host.Arch = runtime.GOARCH - host.SetRole(KubeKey) - - base.AppendHost(host) - base.AppendRoleMap(host) - - local := LocalRuntime{base} - return local, nil -} diff --git a/cmd/kk/pkg/config/from_cluster.go b/cmd/kk/pkg/config/from_cluster.go deleted file mode 100644 index faa63b38e..000000000 --- a/cmd/kk/pkg/config/from_cluster.go +++ /dev/null @@ -1,281 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "bytes" - "context" - "encoding/base64" - "fmt" - "net" - "os" - "os/exec" - "path/filepath" - "strings" - "text/template" - - "github.com/lithammer/dedent" - "github.com/pkg/errors" - "github.com/spf13/viper" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" -) - -var ( - // ClusterCfgTempl defines the template of cluster configuration file for the existing cluster. - ClusterCfgTempl = template.Must(template.New("ClusterCfg").Parse( - dedent.Dedent(`apiVersion: kubekey.kubesphere.io/v1alpha2 -kind: Cluster -metadata: - name: {{ .Options.Name }} -spec: - hosts: - ##You should complete the ssh information of the hosts - {{- range .Options.Hosts }} - - {{ . }} - {{- end }} - roleGroups: - etcd: - - SHOULD_BE_REPLACED - master: - {{- range .Options.MasterGroup }} - - {{ . }} - {{- end }} - worker: - {{- range .Options.WorkerGroup }} - - {{ . }} - {{- end }} - controlPlaneEndpoint: - ##Internal loadbalancer for apiservers - {{- if .Options.InternalLoadbalancer }} - internalLoadbalancer: {{ .Options.InternalLoadbalancer }} - {{- else }} - #internalLoadbalancer: haproxy - {{- end }} - - ##If the external loadbalancer was used, 'address' should be set to loadbalancer's ip. - domain: {{ .Options.ControlPlaneEndpointDomain }} - address: {{ .Options.ControlPlaneEndpointAddress }} - port: {{ .Options.ControlPlaneEndpointPort }} - kubernetes: - version: {{ .Options.KubeVersion }} - clusterName: {{ .Options.ClusterName }} - proxyMode: {{ .Options.ProxyMode }} - masqueradeAll: {{ .Options.MasqueradeAll }} - maxPods: {{ .Options.MaxPods }} - nodeCidrMaskSize: {{ .Options.NodeCidrMaskSize }} - network: - plugin: {{ .Options.NetworkPlugin }} - kubePodsCIDR: {{ .Options.PodNetworkCidr }} - kubeServiceCIDR: {{ .Options.ServiceNetworkCidr }} - registry: - privateRegistry: "" - - `))) -) - -// GenerateClusterCfgStr is used to generate cluster configuration content. -func GenerateClusterCfgStr(opt *OptionsCluster) (string, error) { - return util.Render(ClusterCfgTempl, util.Data{ - "Options": opt, - }) -} - -// OptionsCluster defineds the parameters of cluster configuration for the existing cluster. -type OptionsCluster struct { - Name string - Hosts []string - MasterGroup []string - WorkerGroup []string - KubeVersion string - ImageRepo string - ClusterName string - MasqueradeAll string - ProxyMode string - MaxPods string - NodeCidrMaskSize string - PodNetworkCidr string - ServiceNetworkCidr string - NetworkPlugin string - ControlPlaneEndpointDomain string - ControlPlaneEndpointAddress string - ControlPlaneEndpointPort string - InternalLoadbalancer string -} - -// GetInfoFromCluster is used to fetch information from the existing cluster. -func GetInfoFromCluster(config, name string) (*OptionsCluster, error) { - clientset, err := utils.NewClient(config) - if err != nil { - return nil, err - } - - nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return nil, err - } - - opt := OptionsCluster{} - if name != "" { - output := strings.Split(name, ".") - opt.Name = output[0] - } else { - opt.Name = "config-sample" - } - - for _, node := range nodes.Items { - nodeCfg := kubekeyapiv1alpha2.HostCfg{} - nodeInfo, err := clientset.CoreV1().Nodes().Get(context.TODO(), node.Name, metav1.GetOptions{}) - if err != nil { - return nil, err - } - for _, address := range nodeInfo.Status.Addresses { - if address.Type == "Hostname" { - nodeCfg.Name = address.Address - } - if address.Type == "InternalIP" { - nodeCfg.Address = address.Address - nodeCfg.InternalAddress = address.Address - } - } - if _, ok := nodeInfo.Labels["node-role.kubernetes.io/master"]; ok { - opt.MasterGroup = append(opt.MasterGroup, nodeCfg.Name) - if _, ok := nodeInfo.Labels["node-role.kubernetes.io/worker"]; ok { - opt.WorkerGroup = append(opt.WorkerGroup, nodeCfg.Name) - } - } else { - opt.WorkerGroup = append(opt.WorkerGroup, nodeCfg.Name) - } - nodeCfgStr := fmt.Sprintf("{name: %s, address: %s, internalAddress: %s}", nodeCfg.Name, nodeCfg.Address, nodeCfg.InternalAddress) - opt.Hosts = append(opt.Hosts, nodeCfgStr) - - opt.MaxPods = nodeInfo.Status.Capacity.Pods().String() - } - - kubeadmConfig, err := clientset.CoreV1().ConfigMaps("kube-system").Get(context.TODO(), "kubeadm-config", metav1.GetOptions{}) - if err != nil { - return nil, err - } - - viper.SetConfigType("yaml") - if err := viper.ReadConfig(bytes.NewBuffer([]byte(kubeadmConfig.Data["ClusterConfiguration"]))); err != nil { - return nil, err - } - opt.KubeVersion = viper.GetString("kubernetesVersion") - opt.ImageRepo = viper.GetString("imageRepository") - opt.ClusterName = viper.GetString("clusterName") - opt.PodNetworkCidr = viper.GetString("networking.podSubnet") - opt.ServiceNetworkCidr = viper.GetString("networking.serviceSubnet") - if viper.IsSet("controllerManager.extraArgs.node-cidr-mask-size") { - opt.NodeCidrMaskSize = viper.GetString("controllerManager.extraArgs.node-cidr-mask-size") - } else { - opt.NodeCidrMaskSize = "24" - } - if viper.IsSet("controlPlaneEndpoint") { - controlPlaneEndpointStr := viper.GetString("controlPlaneEndpoint") - strList := strings.Split(controlPlaneEndpointStr, ":") - opt.ControlPlaneEndpointPort = strList[len(strList)-1] - strList = strList[:len(strList)-1] - address := strings.Join(strList, ":") - ip := net.ParseIP(address) - if ip != nil { - opt.ControlPlaneEndpointAddress = address - opt.ControlPlaneEndpointDomain = "lb.kubesphere.local" - } else { - opt.ControlPlaneEndpointAddress = "\"\"" - opt.ControlPlaneEndpointDomain = address - } - } - - pods, err := clientset.CoreV1().Pods("kube-system").List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return nil, err - } - for _, pod := range pods.Items { - if strings.Contains(pod.Name, "calico") { - opt.NetworkPlugin = "calico" - } - if strings.Contains(pod.Name, "flannel") { - opt.NetworkPlugin = "flannel" - } - if strings.Contains(pod.Name, "haproxy-node") { - opt.InternalLoadbalancer = "haproxy" - } - } - - kubeProxyConfig, err := clientset.CoreV1().ConfigMaps("kube-system").Get(context.TODO(), "kube-proxy", metav1.GetOptions{}) - if err != nil { - return nil, err - } - - viper.SetConfigType("yaml") - if err := viper.ReadConfig(bytes.NewBuffer([]byte(kubeProxyConfig.Data["config.conf"]))); err != nil { - return nil, err - } - - opt.MasqueradeAll = viper.GetString("iptables.masqueradeAll") - if viper.GetString("mode") == "ipvs" { - opt.ProxyMode = viper.GetString("mode") - } else { - opt.ProxyMode = "iptables" - } - - return &opt, nil -} - -// GenerateConfigFromCluster is used to generate cluster configuration file from the existing cluster's information. -func GenerateConfigFromCluster(cfgPath, kubeconfig, name string) error { - opt, err := GetInfoFromCluster(kubeconfig, name) - if err != nil { - return err - } - - ClusterCfgStr, err := GenerateClusterCfgStr(opt) - if err != nil { - return errors.Wrap(err, "Failed to generate cluster config") - } - ClusterCfgStrBase64 := base64.StdEncoding.EncodeToString([]byte(ClusterCfgStr)) - var configPath string - - if cfgPath != "" { - CheckConfigFileStatus(cfgPath) - cmd := fmt.Sprintf("echo %s | base64 -d > %s", ClusterCfgStrBase64, cfgPath) - if output, err := exec.Command("/bin/sh", "-c", cmd).Output(); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("Failed to write config to %s: %s", cfgPath, strings.TrimSpace(string(output)))) - } - configPath = cfgPath - } else { - currentDir, err := filepath.Abs(filepath.Dir(os.Args[0])) - if err != nil { - return errors.Wrap(err, "Failed to get current dir") - } - configPath = fmt.Sprintf("%s/%s.yaml", currentDir, opt.Name) - CheckConfigFileStatus(configPath) - cmd := fmt.Sprintf("echo %s | base64 -d > %s/%s.yaml", ClusterCfgStrBase64, currentDir, opt.Name) - err1 := exec.Command("/bin/sh", "-c", cmd).Run() - if err1 != nil { - return err1 - } - - } - notice := "Notice: " + fmt.Sprintf("%s has been created. Some parameters need to be filled in by yourself, please complete it.", configPath) - fmt.Printf("\033[1;36m%s\033[0m\n\n", notice) - return nil -} diff --git a/cmd/kk/pkg/config/generate.go b/cmd/kk/pkg/config/generate.go deleted file mode 100644 index 91a49fcb9..000000000 --- a/cmd/kk/pkg/config/generate.go +++ /dev/null @@ -1,137 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package config - -import ( - "bufio" - "encoding/base64" - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/pkg/errors" - versionutil "k8s.io/apimachinery/pkg/util/version" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/config/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere" -) - -// GenerateKubeKeyConfig is used to generate cluster configuration file -func GenerateKubeKeyConfig(arg common.Argument, name string) error { - if arg.FromCluster { - err := GenerateConfigFromCluster(arg.FilePath, arg.KubeConfig, name) - if err != nil { - return err - } - return nil - } - - opt := new(templates.Options) - - if name != "" { - output := strings.Split(name, ".") - opt.Name = output[0] - } else { - opt.Name = "sample" - } - if len(arg.KubernetesVersion) == 0 { - opt.KubeVersion = kubekeyapiv1alpha2.DefaultKubeVersion - } else { - opt.KubeVersion = arg.KubernetesVersion - } - - if k8sVersion, err := versionutil.ParseGeneric(opt.KubeVersion); err == nil { - if k8sVersion.AtLeast(versionutil.MustParseSemantic("v1.24.0")) { - opt.ContainerManager = common.Containerd - } else { - opt.ContainerManager = common.Docker - } - } - - opt.KubeSphereEnabled = arg.KsEnable - - if arg.KsEnable { - version := strings.TrimSpace(arg.KsVersion) - ksInstaller, ok := kubesphere.StabledVersionSupport(version) - if ok { - opt.KubeSphereConfigMap = ksInstaller.CCToString() - } else if latest, ok := kubesphere.LatestRelease(version); ok { - latest.Version = version - opt.KubeSphereConfigMap = latest.CCToString() - } else if dev, ok := kubesphere.DevRelease(version); ok { - dev.Version = version - opt.KubeSphereConfigMap = dev.CCToString() - } else { - return errors.New(fmt.Sprintf("Unsupported KubeSphere version: %s", version)) - } - } - - ClusterObjStr, err := templates.GenerateCluster(opt) - if err != nil { - return errors.Wrap(err, "Failed to generate cluster config") - } - ClusterObjStrBase64 := base64.StdEncoding.EncodeToString([]byte(ClusterObjStr)) - - if arg.FilePath != "" { - CheckConfigFileStatus(arg.FilePath) - cmdStr := fmt.Sprintf("echo %s | base64 -d > %s", ClusterObjStrBase64, arg.FilePath) - output, err := exec.Command("/bin/sh", "-c", cmdStr).CombinedOutput() - if err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("Failed to write config to %s: %s", arg.FilePath, strings.TrimSpace(string(output)))) - } - } else { - currentDir, err := filepath.Abs(filepath.Dir(os.Args[0])) - if err != nil { - return errors.Wrap(err, "Failed to get current dir") - } - CheckConfigFileStatus(fmt.Sprintf("%s/config-%s.yaml", currentDir, opt.Name)) - cmd := fmt.Sprintf("echo %s | base64 -d > %s/config-%s.yaml", ClusterObjStrBase64, currentDir, opt.Name) - if err := exec.Command("/bin/sh", "-c", cmd).Run(); err != nil { - return err - } - } - - fmt.Println("Generate KubeKey config file successfully") - return nil -} - -// CheckConfigFileStatus is used to check the status of cluster configuration file. -func CheckConfigFileStatus(path string) { - if util.IsExist(path) { - reader := bufio.NewReader(os.Stdin) - Loop: - for { - fmt.Printf("%s already exists. Are you sure you want to overwrite this config file? [yes/no]: ", path) - input, _ := reader.ReadString('\n') - input = strings.TrimSpace(input) - - if input != "" { - switch input { - case "yes": - break Loop - case "no": - os.Exit(0) - } - } - } - } -} diff --git a/cmd/kk/pkg/config/templates/cluster.go b/cmd/kk/pkg/config/templates/cluster.go deleted file mode 100644 index 373d4bd49..000000000 --- a/cmd/kk/pkg/config/templates/cluster.go +++ /dev/null @@ -1,93 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -// Cluster defines the template of cluster configuration file default. -var Cluster = template.Must(template.New("Cluster").Parse( - dedent.Dedent(` -apiVersion: kubekey.kubesphere.io/v1alpha2 -kind: Cluster -metadata: - name: {{ .Options.Name }} -spec: - hosts: - - {name: node1, address: 172.16.0.2, internalAddress: 172.16.0.2, user: ubuntu, password: "Qcloud@123"} - - {name: node2, address: 172.16.0.3, internalAddress: 172.16.0.3, user: ubuntu, password: "Qcloud@123"} - roleGroups: - etcd: - - node1 - control-plane: - - node1 - worker: - - node1 - - node2 - controlPlaneEndpoint: - ## Internal loadbalancer for apiservers - # internalLoadbalancer: haproxy - - domain: lb.kubesphere.local - address: "" - port: 6443 - kubernetes: - version: {{ .Options.KubeVersion }} - clusterName: cluster.local - autoRenewCerts: true - containerManager: {{ .Options.ContainerManager }} - etcd: - type: kubekey - network: - plugin: calico - kubePodsCIDR: 10.233.64.0/18 - kubeServiceCIDR: 10.233.0.0/18 - ## multus support. https://github.com/k8snetworkplumbingwg/multus-cni - multusCNI: - enabled: false - registry: - privateRegistry: "" - namespaceOverride: "" - registryMirrors: [] - insecureRegistries: [] - addons: [] - -{{ if .Options.KubeSphereEnabled }} -{{ .Options.KubeSphereConfigMap }} -{{ end }} - `))) - -// Options defines the parameters of cluster configuration. -type Options struct { - Name string - KubeVersion string - KubeSphereEnabled bool - KubeSphereConfigMap string - ContainerManager string -} - -// GenerateCluster is used to generate cluster configuration content. -func GenerateCluster(opt *Options) (string, error) { - return util.Render(Cluster, util.Data{ - "Options": opt, - }) -} diff --git a/cmd/kk/pkg/container/common.go b/cmd/kk/pkg/container/common.go deleted file mode 100644 index 27e8d057a..000000000 --- a/cmd/kk/pkg/container/common.go +++ /dev/null @@ -1,21 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package container - -const ( - DefaultContainerdCRISocket = "/run/containerd/containerd.sock" -) diff --git a/cmd/kk/pkg/container/containerd.go b/cmd/kk/pkg/container/containerd.go deleted file mode 100644 index 05caae1e9..000000000 --- a/cmd/kk/pkg/container/containerd.go +++ /dev/null @@ -1,530 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package container - -import ( - "fmt" - "path/filepath" - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/registry" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" -) - -type SyncContainerd struct { - common.KubeAction -} - -func (s *SyncContainerd) Execute(runtime connector.Runtime) error { - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) - if !ok { - return errors.New("get KubeBinary by pipeline cache failed") - } - binariesMap := binariesMapObj.(map[string]*files.KubeBinary) - - containerd, ok := binariesMap[common.Containerd] - if !ok { - return errors.New("get KubeBinary key containerd by pipeline cache failed") - } - - dst := filepath.Join(common.TmpDir, containerd.FileName) - if err := runtime.GetRunner().Scp(containerd.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync containerd binaries failed")) - } - - if _, err := runtime.GetRunner().SudoCmd( - fmt.Sprintf("mkdir -p /usr/bin && tar -zxf %s && mv bin/* /usr/bin && rm -rf bin", dst), - false); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("install containerd binaries failed")) - } - return nil -} - -type SyncCrictlBinaries struct { - common.KubeAction -} - -func (s *SyncCrictlBinaries) Execute(runtime connector.Runtime) error { - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) - if !ok { - return errors.New("get KubeBinary by pipeline cache failed") - } - binariesMap := binariesMapObj.(map[string]*files.KubeBinary) - - crictl, ok := binariesMap[common.Crictl] - if !ok { - return errors.New("get KubeBinary key crictl by pipeline cache failed") - } - - dst := filepath.Join(common.TmpDir, crictl.FileName) - - if err := runtime.GetRunner().Scp(crictl.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync crictl binaries failed")) - } - - if _, err := runtime.GetRunner().SudoCmd( - fmt.Sprintf("mkdir -p /usr/bin && tar -zxf %s -C /usr/bin ", dst), - false); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("install crictl binaries failed")) - } - return nil -} - -type EnableContainerd struct { - common.KubeAction -} - -func (e *EnableContainerd) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd( - "systemctl daemon-reload && systemctl enable containerd && systemctl start containerd", - false); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("enable and start containerd failed")) - } - - // install runc - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - binariesMapObj, ok := e.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) - if !ok { - return errors.New("get KubeBinary by pipeline cache failed") - } - binariesMap := binariesMapObj.(map[string]*files.KubeBinary) - - containerd, ok := binariesMap[common.Runc] - if !ok { - return errors.New("get KubeBinary key runc by pipeline cache failed") - } - - dst := filepath.Join(common.TmpDir, containerd.FileName) - if err := runtime.GetRunner().Scp(containerd.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync runc binaries failed")) - } - - if _, err := runtime.GetRunner().SudoCmd( - fmt.Sprintf("install -m 755 %s /usr/local/sbin/runc", dst), - false); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("install runc binaries failed")) - } - return nil -} - -type DisableContainerd struct { - common.KubeAction -} - -func (d *DisableContainerd) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd( - "systemctl disable containerd && systemctl stop containerd", true); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("disable and stop containerd failed")) - } - - // remove containerd related files - files := []string{ - "/usr/local/sbin/runc", - "/usr/bin/crictl", - "/usr/bin/containerd*", - "/usr/bin/ctr", - filepath.Join("/etc/systemd/system", templates.ContainerdService.Name()), - filepath.Join("/etc/containerd", templates.ContainerdConfig.Name()), - filepath.Join("/etc", templates.CrictlConfig.Name()), - } - if d.KubeConf.Cluster.Registry.DataRoot != "" { - files = append(files, d.KubeConf.Cluster.Registry.DataRoot) - } else { - files = append(files, "/var/lib/containerd") - } - - for _, file := range files { - _, _ = runtime.GetRunner().SudoCmd(fmt.Sprintf("rm -rf %s", file), true) - } - return nil -} - -type CordonNode struct { - common.KubeAction -} - -func (d *CordonNode) Execute(runtime connector.Runtime) error { - nodeName := runtime.RemoteHost().GetName() - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("/usr/local/bin/kubectl cordon %s ", nodeName), true); err != nil { - return errors.Wrap(err, fmt.Sprintf("cordon the node: %s failed", nodeName)) - } - return nil -} - -type UnCordonNode struct { - common.KubeAction -} - -func (d *UnCordonNode) Execute(runtime connector.Runtime) error { - nodeName := runtime.RemoteHost().GetName() - f := true - for f { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("/usr/local/bin/kubectl uncordon %s", nodeName), true); err == nil { - break - } - - } - return nil -} - -type DrainNode struct { - common.KubeAction -} - -func (d *DrainNode) Execute(runtime connector.Runtime) error { - nodeName := runtime.RemoteHost().GetName() - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("/usr/local/bin/kubectl drain %s --delete-emptydir-data --ignore-daemonsets --timeout=2m --force", nodeName), true); err != nil { - return errors.Wrap(err, fmt.Sprintf("drain the node: %s failed", nodeName)) - } - return nil -} - -type RestartCri struct { - common.KubeAction -} - -func (i *RestartCri) Execute(runtime connector.Runtime) error { - switch i.KubeConf.Arg.Type { - case common.Docker: - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("systemctl daemon-reload && systemctl restart docker "), true); err != nil { - return errors.Wrap(err, "restart docker") - } - case common.Containerd: - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("systemctl daemon-reload && systemctl restart containerd"), true); err != nil { - return errors.Wrap(err, "restart containerd") - } - - default: - logger.Log.Fatalf("Unsupported container runtime: %s", strings.TrimSpace(i.KubeConf.Arg.Type)) - } - return nil -} - -type EditKubeletCri struct { - common.KubeAction -} - -func (i *EditKubeletCri) Execute(runtime connector.Runtime) error { - switch i.KubeConf.Arg.Type { - case common.Docker: - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "sed -i 's#--container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock --pod#--pod#' /var/lib/kubelet/kubeadm-flags.env"), - true); err != nil { - return errors.Wrap(err, "Change KubeletTo Containerd failed") - } - case common.Containerd: - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "sed -i 's#--network-plugin=cni --pod#--network-plugin=cni --container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock --pod#' /var/lib/kubelet/kubeadm-flags.env"), - true); err != nil { - return errors.Wrap(err, "Change KubeletTo Containerd failed") - } - - default: - logger.Log.Fatalf("Unsupported container runtime: %s", strings.TrimSpace(i.KubeConf.Arg.Type)) - } - return nil -} - -type RestartKubeletNode struct { - common.KubeAction -} - -func (d *RestartKubeletNode) Execute(runtime connector.Runtime) error { - - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("systemctl restart kubelet"), true); err != nil { - return errors.Wrap(err, "RestartNode Kube failed") - } - return nil -} - -func MigrateSelfNodeCriTasks(runtime connector.Runtime, kubeAction common.KubeAction) error { - host := runtime.RemoteHost() - tasks := []task.Interface{} - CordonNode := &task.RemoteTask{ - Name: "CordonNode", - Desc: "Cordon Node", - Hosts: []connector.Host{host}, - - Action: new(CordonNode), - Parallel: false, - } - DrainNode := &task.RemoteTask{ - Name: "DrainNode", - Desc: "Drain Node", - Hosts: []connector.Host{host}, - Action: new(DrainNode), - Parallel: false, - } - RestartCri := &task.RemoteTask{ - Name: "RestartCri", - Desc: "Restart Cri", - Hosts: []connector.Host{host}, - Action: new(RestartCri), - Parallel: false, - } - EditKubeletCri := &task.RemoteTask{ - Name: "EditKubeletCri", - Desc: "Edit Kubelet Cri", - Hosts: []connector.Host{host}, - Action: new(EditKubeletCri), - Parallel: false, - } - RestartKubeletNode := &task.RemoteTask{ - Name: "RestartKubeletNode", - Desc: "Restart Kubelet Node", - Hosts: []connector.Host{host}, - Action: new(RestartKubeletNode), - Parallel: false, - } - UnCordonNode := &task.RemoteTask{ - Name: "UnCordonNode", - Desc: "UnCordon Node", - Hosts: []connector.Host{host}, - Action: new(UnCordonNode), - Parallel: false, - } - switch kubeAction.KubeConf.Cluster.Kubernetes.ContainerManager { - case common.Docker: - Uninstall := &task.RemoteTask{ - Name: "DisableDocker", - Desc: "Disable docker", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - &DockerExist{Not: false}, - }, - Action: new(DisableDocker), - Parallel: false, - } - tasks = append(tasks, CordonNode, DrainNode, Uninstall) - case common.Containerd: - Uninstall := &task.RemoteTask{ - Name: "UninstallContainerd", - Desc: "Uninstall containerd", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - &ContainerdExist{Not: false}, - }, - Action: new(DisableContainerd), - Parallel: false, - } - tasks = append(tasks, CordonNode, DrainNode, Uninstall) - } - if kubeAction.KubeConf.Arg.Type == common.Docker { - syncBinaries := &task.RemoteTask{ - Name: "SyncDockerBinaries", - Desc: "Sync docker binaries", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - // &kubernetes.NodeInCluster{Not: true}, - &DockerExist{Not: true}, - }, - Action: new(SyncDockerBinaries), - Parallel: false, - } - generateDockerService := &task.RemoteTask{ - Name: "GenerateDockerService", - Desc: "Generate docker service", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - // &kubernetes.NodeInCluster{Not: true}, - &DockerExist{Not: true}, - }, - Action: &action.Template{ - Template: templates.DockerService, - Dst: filepath.Join("/etc/systemd/system", templates.DockerService.Name()), - }, - Parallel: false, - } - generateDockerConfig := &task.RemoteTask{ - Name: "GenerateDockerConfig", - Desc: "Generate docker config", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - // &kubernetes.NodeInCluster{Not: true}, - &DockerExist{Not: true}, - }, - Action: &action.Template{ - Template: templates.DockerConfig, - Dst: filepath.Join("/etc/docker/", templates.DockerConfig.Name()), - Data: util.Data{ - "Mirrors": templates.Mirrors(kubeAction.KubeConf), - "InsecureRegistries": templates.InsecureRegistries(kubeAction.KubeConf), - "DataRoot": templates.DataRoot(kubeAction.KubeConf), - }, - }, - Parallel: false, - } - enableDocker := &task.RemoteTask{ - Name: "EnableDocker", - Desc: "Enable docker", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - // &kubernetes.NodeInCluster{Not: true}, - &DockerExist{Not: true}, - }, - Action: new(EnableDocker), - Parallel: false, - } - dockerLoginRegistry := &task.RemoteTask{ - Name: "Login PrivateRegistry", - Desc: "Add auths to container runtime", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - // &kubernetes.NodeInCluster{Not: true}, - &DockerExist{}, - &PrivateRegistryAuth{}, - }, - Action: new(DockerLoginRegistry), - Parallel: false, - } - - tasks = append(tasks, syncBinaries, generateDockerService, generateDockerConfig, enableDocker, dockerLoginRegistry, - RestartCri, EditKubeletCri, RestartKubeletNode, UnCordonNode) - } - if kubeAction.KubeConf.Arg.Type == common.Containerd { - syncContainerd := &task.RemoteTask{ - Name: "SyncContainerd", - Desc: "Sync containerd binaries", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - &ContainerdExist{Not: true}, - }, - Action: new(SyncContainerd), - Parallel: false, - } - - syncCrictlBinaries := &task.RemoteTask{ - Name: "SyncCrictlBinaries", - Desc: "Sync crictl binaries", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - &CrictlExist{Not: true}, - }, - Action: new(SyncCrictlBinaries), - Parallel: false, - } - - generateContainerdService := &task.RemoteTask{ - Name: "GenerateContainerdService", - Desc: "Generate containerd service", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - &ContainerdExist{Not: true}, - }, - Action: &action.Template{ - Template: templates.ContainerdService, - Dst: filepath.Join("/etc/systemd/system", templates.ContainerdService.Name()), - }, - Parallel: false, - } - - generateContainerdConfig := &task.RemoteTask{ - Name: "GenerateContainerdConfig", - Desc: "Generate containerd config", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - &ContainerdExist{Not: true}, - }, - Action: &action.Template{ - Template: templates.ContainerdConfig, - Dst: filepath.Join("/etc/containerd/", templates.ContainerdConfig.Name()), - Data: util.Data{ - "Mirrors": templates.Mirrors(kubeAction.KubeConf), - "InsecureRegistries": kubeAction.KubeConf.Cluster.Registry.InsecureRegistries, - "SandBoxImage": images.GetImage(runtime, kubeAction.KubeConf, "pause").ImageName(), - "Auths": registry.DockerRegistryAuthEntries(kubeAction.KubeConf.Cluster.Registry.Auths), - "DataRoot": templates.DataRoot(kubeAction.KubeConf), - }, - }, - Parallel: false, - } - - generateCrictlConfig := &task.RemoteTask{ - Name: "GenerateCrictlConfig", - Desc: "Generate crictl config", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - &ContainerdExist{Not: true}, - }, - Action: &action.Template{ - Template: templates.CrictlConfig, - Dst: filepath.Join("/etc/", templates.CrictlConfig.Name()), - Data: util.Data{ - "Endpoint": kubeAction.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint, - }, - }, - Parallel: false, - } - - enableContainerd := &task.RemoteTask{ - Name: "EnableContainerd", - Desc: "Enable containerd", - Hosts: []connector.Host{host}, - // Prepare: &prepare.PrepareCollection{ - // &ContainerdExist{Not: true}, - // }, - Action: new(EnableContainerd), - Parallel: false, - } - tasks = append(tasks, syncContainerd, syncCrictlBinaries, generateContainerdService, generateContainerdConfig, - generateCrictlConfig, enableContainerd, RestartCri, EditKubeletCri, RestartKubeletNode, UnCordonNode) - } - - for i := range tasks { - t := tasks[i] - t.Init(runtime, kubeAction.ModuleCache, kubeAction.PipelineCache) - if res := t.Execute(); res.IsFailed() { - return res.CombineErr() - } - } - return nil -} - -type MigrateSelfNodeCri struct { - common.KubeAction -} - -func (d *MigrateSelfNodeCri) Execute(runtime connector.Runtime) error { - - if err := MigrateSelfNodeCriTasks(runtime, d.KubeAction); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("MigrateSelfNodeCriTasks failed:")) - } - return nil -} diff --git a/cmd/kk/pkg/container/docker.go b/cmd/kk/pkg/container/docker.go deleted file mode 100644 index 0beecff3f..000000000 --- a/cmd/kk/pkg/container/docker.go +++ /dev/null @@ -1,148 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package container - -import ( - "fmt" - "path/filepath" - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/registry" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" -) - -type SyncDockerBinaries struct { - common.KubeAction -} - -func (s *SyncDockerBinaries) Execute(runtime connector.Runtime) error { - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) - if !ok { - return errors.New("get KubeBinary by pipeline cache failed") - } - binariesMap := binariesMapObj.(map[string]*files.KubeBinary) - - docker, ok := binariesMap[common.Docker] - if !ok { - return errors.New("get KubeBinary key docker by pipeline cache failed") - } - - dst := filepath.Join(common.TmpDir, docker.FileName) - if err := runtime.GetRunner().Scp(docker.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync docker binaries failed")) - } - - if _, err := runtime.GetRunner().SudoCmd( - fmt.Sprintf("mkdir -p /usr/bin && tar -zxf %s && mv docker/* /usr/bin && rm -rf docker", dst), - false); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("install container runtime docker binaries failed")) - } - return nil -} - -type EnableDocker struct { - common.KubeAction -} - -func (e *EnableDocker) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd( - "systemctl daemon-reload && systemctl enable docker && systemctl start docker", - false); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("enable and start docker failed")) - } - return nil -} - -type DockerLoginRegistry struct { - common.KubeAction -} - -func (p *DockerLoginRegistry) Execute(runtime connector.Runtime) error { - - auths := registry.DockerRegistryAuthEntries(p.KubeConf.Cluster.Registry.Auths) - - for repo, entry := range auths { - if len(entry.Username) == 0 || len(entry.Password) == 0 { - continue - } - cmd := fmt.Sprintf("HOME=$HOME docker login --username '%s' --password '%s' %s", escapeSpecialCharacters(entry.Username), escapeSpecialCharacters(entry.Password), repo) - if _, err := runtime.GetRunner().SudoCmd(cmd, false); err != nil { - return errors.Wrapf(err, "login registry failed, cmd: %v, err:%v", cmd, err) - } - } - - if output, err := runtime.GetRunner().SudoCmd( - "if [ -e $HOME/.docker/config.json ]; "+ - "then echo 'exist'; "+ - "fi", false); err == nil && strings.Contains(output, "exist") { - - cmd := "mkdir -p /.docker && cp -f $HOME/.docker/config.json /.docker/ && chmod 0644 /.docker/config.json " - if _, err := runtime.GetRunner().SudoCmd(cmd, false); err != nil { - return errors.Wrapf(err, "copy docker auths failed cmd: %v, err:%v", cmd, err) - } - } - - return nil -} - -type DisableDocker struct { - common.KubeAction -} - -func (d *DisableDocker) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("systemctl disable docker && systemctl stop docker", - false); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("disable and stop docker failed")) - } - - // remove docker related files - files := []string{ - "/usr/bin/runc", - "/usr/bin/ctr", - "/usr/bin/docker*", - "/usr/bin/containerd*", - filepath.Join("/etc/systemd/system", templates.DockerService.Name()), - filepath.Join("/etc/docker", templates.DockerConfig.Name()), - } - if d.KubeConf.Cluster.Registry.DataRoot != "" { - files = append(files, d.KubeConf.Cluster.Registry.DataRoot) - } else { - files = append(files, "/var/lib/docker") - } - - for _, file := range files { - _, _ = runtime.GetRunner().SudoCmd(fmt.Sprintf("rm -rf %s", file), true) - } - return nil -} - -func escapeSpecialCharacters(str string) string { - newStr := strings.ReplaceAll(str, "$", "\\$") - newStr = strings.ReplaceAll(newStr, "&", "\\&") - newStr = strings.ReplaceAll(newStr, "*", "\\*") - return newStr -} diff --git a/cmd/kk/pkg/container/module.go b/cmd/kk/pkg/container/module.go deleted file mode 100644 index 3cdab21a3..000000000 --- a/cmd/kk/pkg/container/module.go +++ /dev/null @@ -1,388 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package container - -import ( - "path/filepath" - "strings" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/registry" -) - -type InstallContainerModule struct { - common.KubeModule - Skip bool -} - -func (i *InstallContainerModule) IsSkip() bool { - return i.Skip -} - -func (i *InstallContainerModule) Init() { - i.Name = "InstallContainerModule" - i.Desc = "Install container manager" - - switch i.KubeConf.Cluster.Kubernetes.ContainerManager { - case common.Docker: - i.Tasks = InstallDocker(i) - case common.Containerd: - i.Tasks = InstallContainerd(i) - case common.Crio: - // TODO: Add the steps of cri-o's installation. - case common.Isula: - // TODO: Add the steps of iSula's installation. - default: - logger.Log.Fatalf("Unsupported container runtime: %s", strings.TrimSpace(i.KubeConf.Cluster.Kubernetes.ContainerManager)) - } -} - -func InstallDocker(m *InstallContainerModule) []task.Interface { - syncBinaries := &task.RemoteTask{ - Name: "SyncDockerBinaries", - Desc: "Sync docker binaries", - Hosts: m.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &kubernetes.NodeInCluster{Not: true}, - &DockerExist{Not: true}, - }, - Action: new(SyncDockerBinaries), - Parallel: true, - Retry: 2, - } - - generateDockerService := &task.RemoteTask{ - Name: "GenerateDockerService", - Desc: "Generate docker service", - Hosts: m.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &kubernetes.NodeInCluster{Not: true}, - &DockerExist{Not: true}, - }, - Action: &action.Template{ - Template: templates.DockerService, - Dst: filepath.Join("/etc/systemd/system", templates.DockerService.Name()), - }, - Parallel: true, - } - - generateDockerConfig := &task.RemoteTask{ - Name: "GenerateDockerConfig", - Desc: "Generate docker config", - Hosts: m.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &kubernetes.NodeInCluster{Not: true}, - &DockerExist{Not: true}, - }, - Action: &action.Template{ - Template: templates.DockerConfig, - Dst: filepath.Join("/etc/docker/", templates.DockerConfig.Name()), - Data: util.Data{ - "Mirrors": templates.Mirrors(m.KubeConf), - "InsecureRegistries": templates.InsecureRegistries(m.KubeConf), - "DataRoot": templates.DataRoot(m.KubeConf), - "BridgeIP": templates.BridgeIP(m.KubeConf), - }, - }, - Parallel: true, - } - - enableDocker := &task.RemoteTask{ - Name: "EnableDocker", - Desc: "Enable docker", - Hosts: m.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &kubernetes.NodeInCluster{Not: true}, - &DockerExist{Not: true}, - }, - Action: new(EnableDocker), - Parallel: true, - } - - dockerLoginRegistry := &task.RemoteTask{ - Name: "Login PrivateRegistry", - Desc: "Add auths to container runtime", - Hosts: m.Runtime.GetAllHosts(), - Prepare: &prepare.PrepareCollection{ - &kubernetes.NodeInCluster{Not: true}, - &DockerExist{}, - &PrivateRegistryAuth{}, - }, - Action: new(DockerLoginRegistry), - Parallel: true, - } - - return []task.Interface{ - syncBinaries, - generateDockerService, - generateDockerConfig, - enableDocker, - dockerLoginRegistry, - } -} - -func InstallContainerd(m *InstallContainerModule) []task.Interface { - syncContainerd := &task.RemoteTask{ - Name: "SyncContainerd", - Desc: "Sync containerd binaries", - Hosts: m.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &kubernetes.NodeInCluster{Not: true}, - &ContainerdExist{Not: true}, - }, - Action: new(SyncContainerd), - Parallel: true, - Retry: 2, - } - - syncCrictlBinaries := &task.RemoteTask{ - Name: "SyncCrictlBinaries", - Desc: "Sync crictl binaries", - Hosts: m.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &kubernetes.NodeInCluster{Not: true}, - &CrictlExist{Not: true}, - }, - Action: new(SyncCrictlBinaries), - Parallel: true, - Retry: 2, - } - - generateContainerdService := &task.RemoteTask{ - Name: "GenerateContainerdService", - Desc: "Generate containerd service", - Hosts: m.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &kubernetes.NodeInCluster{Not: true}, - &ContainerdExist{Not: true}, - }, - Action: &action.Template{ - Template: templates.ContainerdService, - Dst: filepath.Join("/etc/systemd/system", templates.ContainerdService.Name()), - }, - Parallel: true, - } - - generateContainerdConfig := &task.RemoteTask{ - Name: "GenerateContainerdConfig", - Desc: "Generate containerd config", - Hosts: m.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &kubernetes.NodeInCluster{Not: true}, - &ContainerdExist{Not: true}, - }, - Action: &action.Template{ - Template: templates.ContainerdConfig, - Dst: filepath.Join("/etc/containerd/", templates.ContainerdConfig.Name()), - Data: util.Data{ - "Mirrors": templates.Mirrors(m.KubeConf), - "InsecureRegistries": m.KubeConf.Cluster.Registry.InsecureRegistries, - "SandBoxImage": images.GetImage(m.Runtime, m.KubeConf, "pause").ImageName(), - "Auths": registry.DockerRegistryAuthEntries(m.KubeConf.Cluster.Registry.Auths), - "DataRoot": templates.DataRoot(m.KubeConf), - }, - }, - Parallel: true, - } - - generateCrictlConfig := &task.RemoteTask{ - Name: "GenerateCrictlConfig", - Desc: "Generate crictl config", - Hosts: m.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &kubernetes.NodeInCluster{Not: true}, - &ContainerdExist{Not: true}, - }, - Action: &action.Template{ - Template: templates.CrictlConfig, - Dst: filepath.Join("/etc/", templates.CrictlConfig.Name()), - Data: util.Data{ - "Endpoint": m.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint, - }, - }, - Parallel: true, - } - - enableContainerd := &task.RemoteTask{ - Name: "EnableContainerd", - Desc: "Enable containerd", - Hosts: m.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &kubernetes.NodeInCluster{Not: true}, - &ContainerdExist{Not: true}, - }, - Action: new(EnableContainerd), - Parallel: true, - } - - return []task.Interface{ - syncContainerd, - syncCrictlBinaries, - generateContainerdService, - generateContainerdConfig, - generateCrictlConfig, - enableContainerd, - } -} - -type UninstallContainerModule struct { - common.KubeModule - Skip bool -} - -func (i *UninstallContainerModule) IsSkip() bool { - return i.Skip -} - -func (i *UninstallContainerModule) Init() { - i.Name = "UninstallContainerModule" - i.Desc = "Uninstall container manager" - - switch i.KubeConf.Cluster.Kubernetes.ContainerManager { - case common.Docker: - i.Tasks = UninstallDocker(i) - case common.Containerd: - i.Tasks = UninstallContainerd(i) - case common.Crio: - // TODO: Add the steps of cri-o's installation. - case common.Isula: - // TODO: Add the steps of iSula's installation. - default: - logger.Log.Fatalf("Unsupported container runtime: %s", strings.TrimSpace(i.KubeConf.Cluster.Kubernetes.ContainerManager)) - } -} - -func UninstallDocker(m *UninstallContainerModule) []task.Interface { - - disableDocker := &task.RemoteTask{ - Name: "DisableDocker", - Desc: "Disable docker", - Hosts: m.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &DockerExist{Not: false}, - }, - Action: new(DisableDocker), - Parallel: true, - } - - return []task.Interface{ - disableDocker, - } -} - -func UninstallContainerd(m *UninstallContainerModule) []task.Interface { - disableContainerd := &task.RemoteTask{ - Name: "UninstallContainerd", - Desc: "Uninstall containerd", - Hosts: m.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &ContainerdExist{Not: false}, - }, - Action: new(DisableContainerd), - Parallel: true, - } - - return []task.Interface{ - disableContainerd, - } -} - -type CriMigrateModule struct { - common.KubeModule - - Skip bool -} - -func (i *CriMigrateModule) IsSkip() bool { - return i.Skip -} - -func (p *CriMigrateModule) Init() { - p.Name = "CriMigrateModule" - p.Desc = "Cri Migrate manager" - - if p.KubeConf.Arg.Role == common.Worker { - p.Tasks = MigrateWCri(p) - } else if p.KubeConf.Arg.Role == common.Master { - p.Tasks = MigrateMCri(p) - } else if p.KubeConf.Arg.Role == "all" { - p.Tasks = MigrateACri(p) - } else { - logger.Log.Fatalf("Unsupported Role: %s", strings.TrimSpace(p.KubeConf.Arg.Role)) - } -} - -func MigrateWCri(p *CriMigrateModule) []task.Interface { - - MigrateWCri := &task.RemoteTask{ - Name: "MigrateToDocker", - Desc: "Migrate To Docker", - Hosts: p.Runtime.GetHostsByRole(common.Worker), - Prepare: new(common.OnlyWorker), - Action: new(MigrateSelfNodeCri), - Parallel: false, - } - - p.Tasks = []task.Interface{ - MigrateWCri, - } - - return p.Tasks -} - -func MigrateMCri(p *CriMigrateModule) []task.Interface { - - MigrateMCri := &task.RemoteTask{ - Name: "MigrateMasterToDocker", - Desc: "Migrate Master To Docker", - Hosts: p.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.IsMaster), - Action: new(MigrateSelfNodeCri), - Parallel: false, - } - - p.Tasks = []task.Interface{ - MigrateMCri, - } - - return p.Tasks -} - -func MigrateACri(p *CriMigrateModule) []task.Interface { - - MigrateACri := &task.RemoteTask{ - Name: "MigrateMasterToDocker", - Desc: "Migrate Master To Docker", - Hosts: p.Runtime.GetHostsByRole(common.K8s), - Action: new(MigrateSelfNodeCri), - Parallel: false, - } - - p.Tasks = []task.Interface{ - MigrateACri, - } - - return p.Tasks -} diff --git a/cmd/kk/pkg/container/prepares.go b/cmd/kk/pkg/container/prepares.go deleted file mode 100644 index 34237857d..000000000 --- a/cmd/kk/pkg/container/prepares.go +++ /dev/null @@ -1,92 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package container - -import ( - "strings" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type DockerExist struct { - common.KubePrepare - Not bool -} - -func (d *DockerExist) PreCheck(runtime connector.Runtime) (bool, error) { - output, err := runtime.GetRunner().SudoCmd("if [ -z $(which docker) ] || [ ! -e /var/run/docker.sock ]; "+ - "then echo 'not exist'; "+ - "fi", false) - if err != nil { - return false, err - } - if strings.Contains(output, "not exist") { - return d.Not, nil - } - return !d.Not, nil -} - -type CrictlExist struct { - common.KubePrepare - Not bool -} - -func (c *CrictlExist) PreCheck(runtime connector.Runtime) (bool, error) { - output, err := runtime.GetRunner().SudoCmd( - "if [ -z $(which crictl) ]; "+ - "then echo 'not exist'; "+ - "fi", false) - if err != nil { - return false, err - } - if strings.Contains(output, "not exist") { - return c.Not, nil - } else { - return !c.Not, nil - } -} - -type ContainerdExist struct { - common.KubePrepare - Not bool -} - -func (c *ContainerdExist) PreCheck(runtime connector.Runtime) (bool, error) { - output, err := runtime.GetRunner().SudoCmd( - "if [ -z $(which containerd) ] || [ ! -e /run/containerd/containerd.sock ]; "+ - "then echo 'not exist'; "+ - "fi", false) - if err != nil { - return false, err - } - if strings.Contains(output, "not exist") { - return c.Not, nil - } - return !c.Not, nil -} - -type PrivateRegistryAuth struct { - common.KubePrepare -} - -func (p *PrivateRegistryAuth) PreCheck(runtime connector.Runtime) (bool, error) { - if len(p.KubeConf.Cluster.Registry.Auths.Raw) == 0 { - return false, nil - } - return true, nil -} diff --git a/cmd/kk/pkg/container/templates/containerd_config.go b/cmd/kk/pkg/container/templates/containerd_config.go deleted file mode 100644 index 1d2739ba7..000000000 --- a/cmd/kk/pkg/container/templates/containerd_config.go +++ /dev/null @@ -1,104 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -var ContainerdConfig = template.Must(template.New("config.toml").Parse( - dedent.Dedent(`version = 2 -{{- if .DataRoot }} -root = {{ .DataRoot }} -{{ else }} -root = "/var/lib/containerd" -{{- end }} -state = "/run/containerd" - -[grpc] - address = "/run/containerd/containerd.sock" - uid = 0 - gid = 0 - max_recv_message_size = 16777216 - max_send_message_size = 16777216 - -[ttrpc] - address = "" - uid = 0 - gid = 0 - -[debug] - address = "" - uid = 0 - gid = 0 - level = "" - -[metrics] - address = "" - grpc_histogram = false - -[cgroup] - path = "" - -[timeouts] - "io.containerd.timeout.shim.cleanup" = "5s" - "io.containerd.timeout.shim.load" = "5s" - "io.containerd.timeout.shim.shutdown" = "3s" - "io.containerd.timeout.task.state" = "2s" - -[plugins] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri"] - sandbox_image = "{{ .SandBoxImage }}" - [plugins."io.containerd.grpc.v1.cri".cni] - bin_dir = "/opt/cni/bin" - conf_dir = "/etc/cni/net.d" - max_conf_num = 1 - conf_template = "" - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - {{- if .Mirrors }} - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = [{{ .Mirrors }}, "https://registry-1.docker.io"] - {{ else }} - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - {{- end}} - {{- range $value := .InsecureRegistries }} - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$value}}"] - endpoint = ["http://{{$value}}"] - {{- end}} - - {{- if .Auths }} - [plugins."io.containerd.grpc.v1.cri".registry.configs] - {{- range $repo, $entry := .Auths }} - [plugins."io.containerd.grpc.v1.cri".registry.configs."{{$repo}}".auth] - username = "{{$entry.Username}}" - password = "{{$entry.Password}}" - [plugins."io.containerd.grpc.v1.cri".registry.configs."{{$repo}}".tls] - ca_file = "{{$entry.CAFile}}" - cert_file = "{{$entry.CertFile}}" - key_file = "{{$entry.KeyFile}}" - insecure_skip_verify = {{$entry.SkipTLSVerify}} - {{- end}} - {{- end}} - `))) diff --git a/cmd/kk/pkg/container/templates/containerd_service.go b/cmd/kk/pkg/container/templates/containerd_service.go deleted file mode 100644 index 366ff73e2..000000000 --- a/cmd/kk/pkg/container/templates/containerd_service.go +++ /dev/null @@ -1,51 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var ContainerdService = template.Must(template.New("containerd.service").Parse( - dedent.Dedent(`[Unit] -Description=containerd container runtime -Documentation=https://containerd.io -After=network.target local-fs.target - -[Service] -ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/bin/containerd - -Type=notify -Delegate=yes -KillMode=process -Restart=always -RestartSec=5 -# Having non-zero Limit*s causes performance problems due to accounting overhead -# in the kernel. We recommend using cgroups to do container-local accounting. -LimitNPROC=infinity -LimitCORE=infinity -LimitNOFILE=1048576 -# Comment TasksMax if your systemd version does not supports it. -# Only systemd 226 and above support this version. -TasksMax=infinity -OOMScoreAdjust=-999 - -[Install] -WantedBy=multi-user.target - `))) diff --git a/cmd/kk/pkg/container/templates/crictl_config.go b/cmd/kk/pkg/container/templates/crictl_config.go deleted file mode 100644 index 885d60ae7..000000000 --- a/cmd/kk/pkg/container/templates/crictl_config.go +++ /dev/null @@ -1,30 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var CrictlConfig = template.Must(template.New("crictl.yaml").Parse( - dedent.Dedent(`runtime-endpoint: {{ .Endpoint }} -image-endpoint: {{ .Endpoint }} -timeout: 5 -debug: false -pull-image-on-create: false - `))) diff --git a/cmd/kk/pkg/container/templates/docker_config.go b/cmd/kk/pkg/container/templates/docker_config.go deleted file mode 100644 index 069ea979e..000000000 --- a/cmd/kk/pkg/container/templates/docker_config.go +++ /dev/null @@ -1,106 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "fmt" - "net" - "strings" - "text/template" - - "github.com/lithammer/dedent" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" -) - -var DockerConfig = template.Must(template.New("daemon.json").Parse( - dedent.Dedent(`{ - "log-opts": { - "max-size": "5m", - "max-file":"3" - }, - {{- if .DataRoot }} - "data-root": {{ .DataRoot }}, - {{- end}} - {{- if .Mirrors }} - "registry-mirrors": [{{ .Mirrors }}], - {{- end}} - {{- if .InsecureRegistries }} - "insecure-registries": [{{ .InsecureRegistries }}], - {{- end}} - {{- if .BridgeIP }} - "bip": {{ .BridgeIP }}, - {{- end}} - "exec-opts": ["native.cgroupdriver=systemd"] -} - `))) - -func Mirrors(kubeConf *common.KubeConf) string { - var mirrors string - if kubeConf.Cluster.Registry.RegistryMirrors != nil { - var mirrorsArr []string - for _, mirror := range kubeConf.Cluster.Registry.RegistryMirrors { - mirrorsArr = append(mirrorsArr, fmt.Sprintf("\"%s\"", mirror)) - } - mirrors = strings.Join(mirrorsArr, ", ") - } - return mirrors -} - -func InsecureRegistries(kubeConf *common.KubeConf) string { - var insecureRegistries string - if kubeConf.Cluster.Registry.InsecureRegistries != nil { - var registriesArr []string - for _, repo := range kubeConf.Cluster.Registry.InsecureRegistries { - registriesArr = append(registriesArr, fmt.Sprintf("\"%s\"", repo)) - } - insecureRegistries = strings.Join(registriesArr, ", ") - } - return insecureRegistries -} - -func DataRoot(kubeConf *common.KubeConf) string { - var dataRoot string - if kubeConf.Cluster.Registry.DataRoot != "" { - dataRoot = fmt.Sprintf("\"%s\"", kubeConf.Cluster.Registry.DataRoot) - } - return dataRoot -} - -func BridgeIP(kubeConf *common.KubeConf) string { - var bip string - if kubeConf.Cluster.Registry.BridgeIP != "" { - bip = "172.17.0.1/16" - _, cidr, err := net.ParseCIDR(kubeConf.Cluster.Registry.BridgeIP) - if err != nil { - return bip - } - ip4 := cidr.IP.To4() - if ip4 == nil { - return bip - } - bridge0 := net.IPv4(ip4[0], ip4[1], ip4[2], ip4[3]+1) - if !cidr.Contains(bridge0) { - return bip - } - ones, _ := cidr.Mask.Size() - - bip = fmt.Sprintf("\"%s/%d\"", bridge0.String(), ones) - } - - return bip -} diff --git a/cmd/kk/pkg/container/templates/docker_service.go b/cmd/kk/pkg/container/templates/docker_service.go deleted file mode 100644 index 55dfc7dd8..000000000 --- a/cmd/kk/pkg/container/templates/docker_service.go +++ /dev/null @@ -1,73 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var DockerService = template.Must(template.New("docker.service").Parse( - dedent.Dedent(`[Unit] -Description=Docker Application Container Engine -Documentation=https://docs.docker.com -# After=network-online.target firewalld.service containerd.service -# Wants=network-online.target -# Requires=docker.socket containerd.service - -[Service] -Type=notify -# the default is not to use systemd for cgroups because the delegate issues still -# exists and systemd currently does not support the cgroup feature set required -# for containers run by docker -ExecStart=/usr/bin/dockerd -ExecReload=/bin/kill -s HUP $MAINPID -TimeoutSec=0 -RestartSec=2 -Restart=always - -# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229. -# Both the old, and new location are accepted by systemd 229 and up, so using the old location -# to make them work for either version of systemd. -StartLimitBurst=3 - -# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230. -# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make -# this option work for either version of systemd. -StartLimitInterval=60s - -# Having non-zero Limit*s causes performance problems due to accounting overhead -# in the kernel. We recommend using cgroups to do container-local accounting. -LimitNOFILE=infinity -LimitNPROC=infinity -LimitCORE=infinity - -# Comment TasksMax if your systemd version does not support it. -# Only systemd 226 and above support this option. -TasksMax=infinity - -# set delegate yes so that systemd does not reset the cgroups of docker containers -Delegate=yes - -# kill only the docker process, not all processes in the cgroup -KillMode=process -OOMScoreAdjust=-500 - -[Install] -WantedBy=multi-user.target - - `))) diff --git a/cmd/kk/pkg/core/action/base.go b/cmd/kk/pkg/core/action/base.go deleted file mode 100644 index 0d09b5497..000000000 --- a/cmd/kk/pkg/core/action/base.go +++ /dev/null @@ -1,40 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package action - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type BaseAction struct { - ModuleCache *cache.Cache - PipelineCache *cache.Cache -} - -func (b *BaseAction) Init(moduleCache *cache.Cache, pipelineCache *cache.Cache) { - b.ModuleCache = moduleCache - b.PipelineCache = pipelineCache -} - -func (b *BaseAction) Execute(runtime connector.Runtime) error { - return nil -} - -func (b *BaseAction) AutoAssert(runtime connector.Runtime) { - -} diff --git a/cmd/kk/pkg/core/action/interface.go b/cmd/kk/pkg/core/action/interface.go deleted file mode 100644 index cc1fd45fe..000000000 --- a/cmd/kk/pkg/core/action/interface.go +++ /dev/null @@ -1,28 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package action - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type Action interface { - Execute(runtime connector.Runtime) (err error) - Init(cache *cache.Cache, rootCache *cache.Cache) - AutoAssert(runtime connector.Runtime) -} diff --git a/cmd/kk/pkg/core/action/template.go b/cmd/kk/pkg/core/action/template.go deleted file mode 100644 index cde5d1d22..000000000 --- a/cmd/kk/pkg/core/action/template.go +++ /dev/null @@ -1,53 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package action - -import ( - "fmt" - "path/filepath" - "text/template" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -type Template struct { - BaseAction - Template *template.Template - Dst string - Data util.Data -} - -func (t *Template) Execute(runtime connector.Runtime) error { - templateStr, err := util.Render(t.Template, t.Data) - if err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("render template %s failed", t.Template.Name())) - } - - fileName := filepath.Join(runtime.GetHostWorkDir(), t.Template.Name()) - if err := util.WriteFile(fileName, []byte(templateStr)); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("write file %s failed", fileName)) - } - - if err := runtime.GetRunner().SudoScp(fileName, t.Dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("scp file %s to remote %s failed", fileName, t.Dst)) - } - - return nil -} diff --git a/cmd/kk/pkg/core/cache/cache.go b/cmd/kk/pkg/core/cache/cache.go deleted file mode 100644 index fc5d20a75..000000000 --- a/cmd/kk/pkg/core/cache/cache.go +++ /dev/null @@ -1,85 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cache - -import "sync" - -type Cache struct { - store sync.Map -} - -func NewCache() *Cache { - var m Cache - return &m -} - -func (c *Cache) Set(k string, v interface{}) { - c.store.Store(k, v) -} - -// GetOrSet returns the existing value for the key if present. -// Otherwise, it stores and returns the given value. -// The loaded result is true if the value was loaded, false if stored. -func (c *Cache) GetOrSet(k string, v interface{}) (interface{}, bool) { - return c.store.LoadOrStore(k, v) -} - -func (c *Cache) Get(k string) (interface{}, bool) { - return c.store.Load(k) -} - -func (c *Cache) Range(f func(key, value interface{}) bool) { - c.store.Range(f) -} - -func (c *Cache) Delete(k string) { - c.store.Delete(k) -} - -func (c *Cache) Clean() { - c.store.Range(func(key, value interface{}) bool { - c.store.Delete(key) - return true - }) -} - -func (c *Cache) GetMustInt(k string) (int, bool) { - v, ok := c.Get(k) - res, assert := v.(int) - if !assert { - return res, false - } - return res, ok -} - -func (c *Cache) GetMustString(k string) (string, bool) { - v, ok := c.Get(k) - res, assert := v.(string) - if !assert { - return res, false - } - return res, ok -} - -func (c *Cache) GetMustBool(k string) (bool, bool) { - v, ok := c.Get(k) - res, assert := v.(bool) - if !assert { - return res, false - } - return res, ok -} diff --git a/cmd/kk/pkg/core/common/common.go b/cmd/kk/pkg/core/common/common.go deleted file mode 100644 index 5d42ec456..000000000 --- a/cmd/kk/pkg/core/common/common.go +++ /dev/null @@ -1,37 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -const ( - KubeKey = "kubekey" - - Pipeline = "Pipeline" - Module = "Module" - Task = "Task" - Node = "Node" - - LocalHost = "LocalHost" - - FileMode0755 = 0755 - FileMode0644 = 0644 - - TmpDir = "/tmp/kubekey/" - - // command - CopyCmd = "cp -r %s %s" - MoveCmd = "mv -f %s %s" -) diff --git a/cmd/kk/pkg/core/connector/dialer.go b/cmd/kk/pkg/core/connector/dialer.go deleted file mode 100644 index 5fc77e35c..000000000 --- a/cmd/kk/pkg/core/connector/dialer.go +++ /dev/null @@ -1,86 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package connector - -import ( - "sync" - "time" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" -) - -type Dialer struct { - lock sync.Mutex - connections map[string]Connection -} - -func NewDialer() *Dialer { - return &Dialer{ - connections: make(map[string]Connection), - } -} - -func (d *Dialer) Connect(host Host) (Connection, error) { - var err error - - d.lock.Lock() - defer d.lock.Unlock() - - conn, ok := d.connections[host.GetName()] - if !ok { - opts := Cfg{ - Username: host.GetUser(), - Port: host.GetPort(), - Address: host.GetAddress(), - Password: host.GetPassword(), - PrivateKey: host.GetPrivateKey(), - KeyFile: host.GetPrivateKeyPath(), - Timeout: time.Duration(host.GetTimeout()) * time.Second, - } - conn, err = NewConnection(opts) - if err != nil { - return nil, err - } - d.connections[host.GetName()] = conn - } - - return conn, nil -} - -func (d *Dialer) Close(host Host) { - conn, ok := d.connections[host.GetName()] - if !ok { - return - } - - conn.Close() - logger.Log.Debugf("close connection %s", host.GetName()) - - c := conn.(*connection) - d.forgetConnection(c) -} - -func (d *Dialer) forgetConnection(conn *connection) { - d.lock.Lock() - defer d.lock.Unlock() - - for k := range d.connections { - if d.connections[k] == conn { - delete(d.connections, k) - } - } -} diff --git a/cmd/kk/pkg/core/connector/host.go b/cmd/kk/pkg/core/connector/host.go deleted file mode 100644 index 762c5a89f..000000000 --- a/cmd/kk/pkg/core/connector/host.go +++ /dev/null @@ -1,154 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package connector - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" -) - -type BaseHost struct { - Name string `yaml:"name,omitempty" json:"name,omitempty"` - Address string `yaml:"address,omitempty" json:"address,omitempty"` - InternalAddress string `yaml:"internalAddress,omitempty" json:"internalAddress,omitempty"` - Port int `yaml:"port,omitempty" json:"port,omitempty"` - User string `yaml:"user,omitempty" json:"user,omitempty"` - Password string `yaml:"password,omitempty" json:"password,omitempty"` - PrivateKey string `yaml:"privateKey,omitempty" json:"privateKey,omitempty"` - PrivateKeyPath string `yaml:"privateKeyPath,omitempty" json:"privateKeyPath,omitempty"` - Arch string `yaml:"arch,omitempty" json:"arch,omitempty"` - Timeout int64 `yaml:"timeout,omitempty" json:"timeout,omitempty"` - - Roles []string `json:"-"` - RoleTable map[string]bool `json:"-"` - Cache *cache.Cache `json:"-"` -} - -func NewHost() *BaseHost { - return &BaseHost{ - Roles: make([]string, 0, 0), - RoleTable: make(map[string]bool), - Cache: cache.NewCache(), - } -} - -func (b *BaseHost) GetName() string { - return b.Name -} - -func (b *BaseHost) SetName(name string) { - b.Name = name -} - -func (b *BaseHost) GetAddress() string { - return b.Address -} - -func (b *BaseHost) SetAddress(str string) { - b.Address = str -} - -func (b *BaseHost) GetInternalAddress() string { - return b.InternalAddress -} - -func (b *BaseHost) SetInternalAddress(str string) { - b.InternalAddress = str -} - -func (b *BaseHost) GetPort() int { - return b.Port -} - -func (b *BaseHost) SetPort(port int) { - b.Port = port -} - -func (b *BaseHost) GetUser() string { - return b.User -} - -func (b *BaseHost) SetUser(u string) { - b.User = u -} - -func (b *BaseHost) GetPassword() string { - return b.Password -} - -func (b *BaseHost) SetPassword(password string) { - b.Password = password -} - -func (b *BaseHost) GetPrivateKey() string { - return b.PrivateKey -} - -func (b *BaseHost) SetPrivateKey(privateKey string) { - b.PrivateKey = privateKey -} - -func (b *BaseHost) GetPrivateKeyPath() string { - return b.PrivateKeyPath -} - -func (b *BaseHost) SetPrivateKeyPath(path string) { - b.PrivateKeyPath = path -} - -func (b *BaseHost) GetArch() string { - return b.Arch -} - -func (b *BaseHost) SetArch(arch string) { - b.Arch = arch -} - -func (b *BaseHost) GetTimeout() int64 { - return b.Timeout -} - -func (b *BaseHost) SetTimeout(timeout int64) { - b.Timeout = timeout -} - -func (b *BaseHost) GetRoles() []string { - return b.Roles -} - -func (b *BaseHost) SetRoles(roles []string) { - b.Roles = roles -} - -func (b *BaseHost) SetRole(role string) { - b.RoleTable[role] = true - b.Roles = append(b.Roles, role) -} - -func (b *BaseHost) IsRole(role string) bool { - if res, ok := b.RoleTable[role]; ok { - return res - } - return false -} - -func (b *BaseHost) GetCache() *cache.Cache { - return b.Cache -} - -func (b *BaseHost) SetCache(c *cache.Cache) { - b.Cache = c -} diff --git a/cmd/kk/pkg/core/connector/interface.go b/cmd/kk/pkg/core/connector/interface.go deleted file mode 100644 index 60d124182..000000000 --- a/cmd/kk/pkg/core/connector/interface.go +++ /dev/null @@ -1,93 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package connector - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "io" - "os" -) - -type Connection interface { - Exec(cmd string, host Host) (stdout string, code int, err error) - PExec(cmd string, stdin io.Reader, stdout io.Writer, stderr io.Writer, host Host) (code int, err error) - Fetch(local, remote string, host Host) error - Scp(local, remote string, host Host) error - RemoteFileExist(remote string, host Host) bool - RemoteDirExist(remote string, host Host) (bool, error) - MkDirAll(path string, mode string, host Host) error - Chmod(path string, mode os.FileMode) error - Close() -} - -type Connector interface { - Connect(host Host) (Connection, error) - Close(host Host) -} - -type ModuleRuntime interface { - GetObjName() string - SetObjName(name string) - GenerateWorkDir() error - GetHostWorkDir() string - GetWorkDir() string - GetIgnoreErr() bool - GetAllHosts() []Host - SetAllHosts([]Host) - GetHostsByRole(role string) []Host - DeleteHost(host Host) - HostIsDeprecated(host Host) bool - InitLogger() error -} - -type Runtime interface { - GetRunner() *Runner - SetRunner(r *Runner) - GetConnector() Connector - SetConnector(c Connector) - RemoteHost() Host - Copy() Runtime - ModuleRuntime -} - -type Host interface { - GetName() string - SetName(name string) - GetAddress() string - SetAddress(str string) - GetInternalAddress() string - SetInternalAddress(str string) - GetPort() int - SetPort(port int) - GetUser() string - SetUser(u string) - GetPassword() string - SetPassword(password string) - GetPrivateKey() string - SetPrivateKey(privateKey string) - GetPrivateKeyPath() string - SetPrivateKeyPath(path string) - GetArch() string - SetArch(arch string) - GetTimeout() int64 - SetTimeout(timeout int64) - GetRoles() []string - SetRoles(roles []string) - IsRole(role string) bool - GetCache() *cache.Cache - SetCache(c *cache.Cache) -} diff --git a/cmd/kk/pkg/core/connector/runner.go b/cmd/kk/pkg/core/connector/runner.go deleted file mode 100644 index eea7ccc2b..000000000 --- a/cmd/kk/pkg/core/connector/runner.go +++ /dev/null @@ -1,191 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package connector - -import ( - "errors" - "fmt" - "os" - "path/filepath" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -type Runner struct { - Conn Connection - Debug bool - Host Host - Index int -} - -func (r *Runner) Exec(cmd string, printOutput bool) (string, int, error) { - if r.Conn == nil { - return "", 1, errors.New("no ssh connection available") - } - - stdout, code, err := r.Conn.Exec(cmd, r.Host) - logger.Log.Debugf("command: [%s]\n%s", r.Host.GetName(), cmd) - if stdout != "" { - logger.Log.Debugf("stdout: [%s]\n%s", r.Host.GetName(), stdout) - } - if err != nil { - logger.Log.Debugf("stderr: [%s]\n%s", r.Host.GetName(), err) - } - - if printOutput { - if stdout != "" { - logger.Log.Infof("stdout: [%s]\n%s", r.Host.GetName(), stdout) - } - } - return stdout, code, err -} - -func (r *Runner) Cmd(cmd string, printOutput bool) (string, error) { - stdout, _, err := r.Exec(cmd, printOutput) - if err != nil { - return stdout, err - } - return stdout, nil -} - -func (r *Runner) SudoExec(cmd string, printOutput bool) (string, int, error) { - return r.Exec(SudoPrefix(cmd), printOutput) -} - -func (r *Runner) SudoCmd(cmd string, printOutput bool) (string, error) { - return r.Cmd(SudoPrefix(cmd), printOutput) -} - -func (r *Runner) Fetch(local, remote string) error { - if r.Conn == nil { - return errors.New("no ssh connection available") - } - - if err := r.Conn.Fetch(local, remote, r.Host); err != nil { - logger.Log.Debugf("fetch remote file %s to local %s failed: %v", remote, local, err) - return err - } - logger.Log.Debugf("fetch remote file %s to local %s success", remote, local) - return nil -} - -func (r *Runner) Scp(local, remote string) error { - if r.Conn == nil { - return errors.New("no ssh connection available") - } - - if err := r.Conn.Scp(local, remote, r.Host); err != nil { - logger.Log.Debugf("scp local file %s to remote %s failed: %v", local, remote, err) - return err - } - logger.Log.Debugf("scp local file %s to remote %s success", local, remote) - return nil -} - -func (r *Runner) SudoScp(local, remote string) error { - if r.Conn == nil { - return errors.New("no ssh connection available") - } - - // scp to tmp dir - remoteTmp := filepath.Join(common.TmpDir, remote) - //remoteTmp := remote - if err := r.Scp(local, remoteTmp); err != nil { - return err - } - - baseRemotePath := remote - if !util.IsDir(local) { - baseRemotePath = filepath.Dir(remote) - } - if err := r.Conn.MkDirAll(baseRemotePath, "", r.Host); err != nil { - return err - } - - if _, err := r.SudoCmd(fmt.Sprintf(common.MoveCmd, remoteTmp, remote), false); err != nil { - return err - } - - if _, err := r.SudoCmd(fmt.Sprintf("rm -rf %s", filepath.Join(common.TmpDir, "*")), false); err != nil { - return err - } - return nil -} - -func (r *Runner) FileExist(remote string) (bool, error) { - if r.Conn == nil { - return false, errors.New("no ssh connection available") - } - - ok := r.Conn.RemoteFileExist(remote, r.Host) - logger.Log.Debugf("check remote file exist: %v", ok) - return ok, nil -} - -func (r *Runner) DirExist(remote string) (bool, error) { - if r.Conn == nil { - return false, errors.New("no ssh connection available") - } - - ok, err := r.Conn.RemoteDirExist(remote, r.Host) - if err != nil { - logger.Log.Debugf("check remote dir exist failed: %v", err) - return false, err - } - logger.Log.Debugf("check remote dir exist: %v", ok) - return ok, nil -} - -func (r *Runner) MkDir(path string) error { - if r.Conn == nil { - return errors.New("no ssh connection available") - } - - if err := r.Conn.MkDirAll(path, "", r.Host); err != nil { - logger.Log.Errorf("make remote dir %s failed: %v", path, err) - return err - } - return nil -} - -func (r *Runner) Chmod(path string, mode os.FileMode) error { - if r.Conn == nil { - return errors.New("no ssh connection available") - } - - if err := r.Conn.Chmod(path, mode); err != nil { - logger.Log.Errorf("chmod remote path %s failed: %v", path, err) - return err - } - return nil -} - -func (r *Runner) FileMd5(path string) (string, error) { - if r.Conn == nil { - return "", errors.New("no ssh connection available") - } - - cmd := fmt.Sprintf("md5sum %s | cut -d\" \" -f1", path) - out, _, err := r.Conn.Exec(cmd, r.Host) - if err != nil { - logger.Log.Errorf("count remote %s md5 failed: %v", path, err) - return "", err - } - return out, nil -} diff --git a/cmd/kk/pkg/core/connector/runtime.go b/cmd/kk/pkg/core/connector/runtime.go deleted file mode 100644 index 48d32f0ec..000000000 --- a/cmd/kk/pkg/core/connector/runtime.go +++ /dev/null @@ -1,225 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package connector - -import ( - "fmt" - "os" - "path/filepath" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -type BaseRuntime struct { - ObjName string - connector Connector - runner *Runner - workDir string - verbose bool - ignoreErr bool - allHosts []Host - roleHosts map[string][]Host - deprecatedHosts map[string]string -} - -func NewBaseRuntime(name string, connector Connector, verbose bool, ignoreErr bool) BaseRuntime { - base := BaseRuntime{ - ObjName: name, - connector: connector, - verbose: verbose, - ignoreErr: ignoreErr, - allHosts: make([]Host, 0, 0), - roleHosts: make(map[string][]Host), - deprecatedHosts: make(map[string]string), - } - if err := base.GenerateWorkDir(); err != nil { - fmt.Printf("[ERRO]: Failed to create KubeKey work dir: %s\n", err) - os.Exit(1) - } - if err := base.InitLogger(); err != nil { - fmt.Printf("[ERRO]: Failed to init KubeKey log entry: %s\n", err) - os.Exit(1) - } - return base -} - -func (b *BaseRuntime) GetObjName() string { - return b.ObjName -} - -func (b *BaseRuntime) SetObjName(name string) { - b.ObjName = name -} - -func (b *BaseRuntime) GetRunner() *Runner { - return b.runner -} - -func (b *BaseRuntime) SetRunner(r *Runner) { - b.runner = r -} - -func (b *BaseRuntime) GetConnector() Connector { - return b.connector -} - -func (b *BaseRuntime) SetConnector(c Connector) { - b.connector = c -} - -func (b *BaseRuntime) GenerateWorkDir() error { - currentDir, err := filepath.Abs(filepath.Dir(os.Args[0])) - if err != nil { - return errors.Wrap(err, "get current dir failed") - } - - rootPath := filepath.Join(currentDir, common.KubeKey) - if err := util.CreateDir(rootPath); err != nil { - return errors.Wrap(err, "create work dir failed") - } - b.workDir = rootPath - - logDir := filepath.Join(rootPath, "logs") - if err := util.CreateDir(logDir); err != nil { - return errors.Wrap(err, "create logs dir failed") - } - - for i := range b.allHosts { - subPath := filepath.Join(rootPath, b.allHosts[i].GetName()) - if err := util.CreateDir(subPath); err != nil { - return errors.Wrap(err, "create work dir failed") - } - } - return nil -} - -func (b *BaseRuntime) GetHostWorkDir() string { - return filepath.Join(b.workDir, b.RemoteHost().GetName()) -} - -func (b *BaseRuntime) GetWorkDir() string { - return b.workDir -} - -func (b *BaseRuntime) GetIgnoreErr() bool { - return b.ignoreErr -} - -func (b *BaseRuntime) GetAllHosts() []Host { - hosts := make([]Host, 0, 0) - for i := range b.allHosts { - if b.allHosts[i] == nil || b.HostIsDeprecated(b.allHosts[i]) { - continue - } - hosts = append(hosts, b.allHosts[i]) - } - return hosts -} - -func (b *BaseRuntime) SetAllHosts(hosts []Host) { - b.allHosts = hosts -} - -func (b *BaseRuntime) GetHostsByRole(role string) []Host { - if _, ok := b.roleHosts[role]; ok { - return b.roleHosts[role] - } else { - return []Host{} - } -} - -func (b *BaseRuntime) RemoteHost() Host { - return b.GetRunner().Host -} - -func (b *BaseRuntime) DeleteHost(host Host) { - i := 0 - for j := range b.allHosts { - if b.allHosts[j].GetName() != host.GetName() { - b.allHosts[i] = b.allHosts[j] - i++ - } - } - b.allHosts[i] = nil - b.allHosts = b.allHosts[:i] - b.RoleMapDelete(host) - b.deprecatedHosts[host.GetName()] = "" -} - -func (b *BaseRuntime) HostIsDeprecated(host Host) bool { - if _, ok := b.deprecatedHosts[host.GetName()]; ok { - return true - } - return false -} - -func (b *BaseRuntime) InitLogger() error { - if b.GetWorkDir() == "" { - if err := b.GenerateWorkDir(); err != nil { - return err - } - } - logDir := filepath.Join(b.GetWorkDir(), "logs") - logger.Log = logger.NewLogger(logDir, b.verbose) - return nil -} - -func (b *BaseRuntime) Copy() Runtime { - runtime := *b - return &runtime -} - -func (b *BaseRuntime) GenerateRoleMap() { - for i := range b.allHosts { - b.AppendRoleMap(b.allHosts[i]) - } -} - -func (b *BaseRuntime) AppendHost(host Host) { - b.allHosts = append(b.allHosts, host) -} - -func (b *BaseRuntime) AppendRoleMap(host Host) { - for _, r := range host.GetRoles() { - if hosts, ok := b.roleHosts[r]; ok { - hosts = append(hosts, host) - b.roleHosts[r] = hosts - } else { - first := make([]Host, 0, 0) - first = append(first, host) - b.roleHosts[r] = first - } - } -} - -func (b *BaseRuntime) RoleMapDelete(host Host) { - for role, hosts := range b.roleHosts { - i := 0 - for j := range hosts { - if hosts[j].GetName() != host.GetName() { - hosts[i] = hosts[j] - i++ - } - } - hosts = hosts[:i] - b.roleHosts[role] = hosts - } -} diff --git a/cmd/kk/pkg/core/connector/ssh.go b/cmd/kk/pkg/core/connector/ssh.go deleted file mode 100644 index 261ee7f0c..000000000 --- a/cmd/kk/pkg/core/connector/ssh.go +++ /dev/null @@ -1,608 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package connector - -import ( - "bufio" - "context" - "encoding/base64" - "fmt" - "io" - "net" - "os" - "path" - "path/filepath" - "strconv" - "strings" - "sync" - "time" - - "github.com/pkg/errors" - "github.com/pkg/sftp" - "golang.org/x/crypto/ssh" - "golang.org/x/crypto/ssh/agent" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -type Cfg struct { - Username string - Password string - Address string - Port int - PrivateKey string - KeyFile string - AgentSocket string - Timeout time.Duration - Bastion string - BastionPort int - BastionUser string -} - -const socketEnvPrefix = "env:" - -type connection struct { - mu sync.Mutex - sftpclient *sftp.Client - sshclient *ssh.Client - ctx context.Context - cancel context.CancelFunc -} - -func NewConnection(cfg Cfg) (Connection, error) { - cfg, err := validateOptions(cfg) - if err != nil { - return nil, errors.Wrap(err, "Failed to validate ssh connection parameters") - } - - authMethods := make([]ssh.AuthMethod, 0) - - if len(cfg.Password) > 0 { - authMethods = append(authMethods, ssh.Password(cfg.Password)) - } - - if len(cfg.PrivateKey) > 0 { - signer, parseErr := ssh.ParsePrivateKey([]byte(cfg.PrivateKey)) - if parseErr != nil { - return nil, errors.Wrap(parseErr, "The given SSH key could not be parsed") - } - authMethods = append(authMethods, ssh.PublicKeys(signer)) - } - - if len(cfg.AgentSocket) > 0 { - addr := cfg.AgentSocket - - if strings.HasPrefix(cfg.AgentSocket, socketEnvPrefix) { - envName := strings.TrimPrefix(cfg.AgentSocket, socketEnvPrefix) - - if envAddr := os.Getenv(envName); len(envAddr) > 0 { - addr = envAddr - } - } - - socket, dialErr := net.Dial("unix", addr) - if dialErr != nil { - return nil, errors.Wrapf(dialErr, "could not open socket %q", addr) - } - - agentClient := agent.NewClient(socket) - - signers, signersErr := agentClient.Signers() - if signersErr != nil { - _ = socket.Close() - return nil, errors.Wrap(signersErr, "error when creating signer for SSH agent") - } - - authMethods = append(authMethods, ssh.PublicKeys(signers...)) - } - - sshConfig := &ssh.ClientConfig{ - User: cfg.Username, - Timeout: cfg.Timeout, - Auth: authMethods, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - - targetHost := cfg.Address - targetPort := strconv.Itoa(cfg.Port) - - if cfg.Bastion != "" { - targetHost = cfg.Bastion - targetPort = strconv.Itoa(cfg.BastionPort) - sshConfig.User = cfg.BastionUser - } - - endpoint := net.JoinHostPort(targetHost, targetPort) - - client, err := ssh.Dial("tcp", endpoint, sshConfig) - if err != nil { - return nil, errors.Wrapf(err, "could not establish connection to %s", endpoint) - } - - ctx, cancelFn := context.WithCancel(context.Background()) - sshConn := &connection{ - ctx: ctx, - cancel: cancelFn, - } - - if cfg.Bastion == "" { - sshConn.sshclient = client - sftpClient, err := sftp.NewClient(sshConn.sshclient) - if err != nil { - return nil, errors.Wrapf(err, "new sftp client failed: %v", err) - } - sshConn.sftpclient = sftpClient - return sshConn, nil - } - - endpointBehindBastion := net.JoinHostPort(cfg.Address, strconv.Itoa(cfg.Port)) - - conn, err := client.Dial("tcp", endpointBehindBastion) - if err != nil { - return nil, errors.Wrapf(err, "could not establish connection to %s", endpointBehindBastion) - } - - sshConfig.User = cfg.Username - ncc, chans, reqs, err := ssh.NewClientConn(conn, endpointBehindBastion, sshConfig) - if err != nil { - return nil, errors.Wrapf(err, "could not establish connection to %s", endpointBehindBastion) - } - - sshConn.sshclient = ssh.NewClient(ncc, chans, reqs) - sftpClient, err := sftp.NewClient(sshConn.sshclient) - if err != nil { - return nil, errors.Wrapf(err, "new sftp client failed: %v", err) - } - sshConn.sftpclient = sftpClient - return sshConn, nil -} - -func validateOptions(cfg Cfg) (Cfg, error) { - if len(cfg.Username) == 0 { - return cfg, errors.New("No username specified for SSH connection") - } - - if len(cfg.Address) == 0 { - return cfg, errors.New("No address specified for SSH connection") - } - - if len(cfg.Password) == 0 && len(cfg.PrivateKey) == 0 && len(cfg.KeyFile) == 0 && len(cfg.AgentSocket) == 0 { - return cfg, errors.New("Must specify at least one of password, private key, keyfile or agent socket") - } - - if len(cfg.PrivateKey) == 0 && len(cfg.KeyFile) > 0 { - content, err := os.ReadFile(cfg.KeyFile) - if err != nil { - return cfg, errors.Wrapf(err, "Failed to read keyfile %q", cfg.KeyFile) - } - - cfg.PrivateKey = string(content) - cfg.KeyFile = "" - } - - if cfg.Port <= 0 { - cfg.Port = 22 - } - - if cfg.BastionPort <= 0 { - cfg.BastionPort = 22 - } - - if cfg.BastionUser == "" { - cfg.BastionUser = cfg.Username - } - - if cfg.Timeout == 0 { - cfg.Timeout = 15 * time.Second - } - - return cfg, nil -} - -func (c *connection) Close() { - c.mu.Lock() - defer c.mu.Unlock() - - if c.sshclient == nil && c.sftpclient == nil { - return - } - c.cancel() - - if c.sshclient != nil { - c.sshclient.Close() - c.sshclient = nil - } - if c.sftpclient != nil { - c.sftpclient.Close() - c.sftpclient = nil - } -} - -func (c *connection) session() (*ssh.Session, error) { - c.mu.Lock() - defer c.mu.Unlock() - - if c.sshclient == nil { - return nil, errors.New("connection closed") - } - - sess, err := c.sshclient.NewSession() - if err != nil { - return nil, err - } - - modes := ssh.TerminalModes{ - ssh.ECHO: 0, // disable echoing - ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud - ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud - } - - err = sess.RequestPty("xterm", 100, 50, modes) - if err != nil { - return nil, err - } - - return sess, nil -} - -func (c *connection) PExec(cmd string, stdin io.Reader, stdout io.Writer, stderr io.Writer, host Host) (int, error) { - sess, err := c.session() - if err != nil { - return 1, errors.Wrap(err, "failed to get SSH session") - } - defer sess.Close() - - sess.Stdin = stdin - sess.Stdout = stdout - sess.Stderr = stderr - - exitCode := 0 - - in, _ := sess.StdinPipe() - out, _ := sess.StdoutPipe() - - err = sess.Start(strings.TrimSpace(cmd)) - if err != nil { - exitCode = -1 - if exitErr, ok := err.(*ssh.ExitError); ok { - exitCode = exitErr.ExitStatus() - } - } - - var ( - output []byte - line = "" - r = bufio.NewReader(out) - ) - - for { - b, err := r.ReadByte() - if err != nil { - break - } - - output = append(output, b) - - if b == byte('\n') { - line = "" - continue - } - - line += string(b) - - if (strings.HasPrefix(line, "[sudo] password for ") || strings.HasPrefix(line, "Password")) && strings.HasSuffix(line, ": ") { - _, err = in.Write([]byte(host.GetPassword() + "\n")) - if err != nil { - break - } - } - } - err = sess.Wait() - if err != nil { - exitCode = -1 - if exitErr, ok := err.(*ssh.ExitError); ok { - exitCode = exitErr.ExitStatus() - } - } - - // preserve original error - return exitCode, err -} - -func (c *connection) Exec(cmd string, host Host) (stdout string, code int, err error) { - sess, err := c.session() - if err != nil { - return "", 1, errors.Wrap(err, "failed to get SSH session") - } - defer sess.Close() - - exitCode := 0 - - in, _ := sess.StdinPipe() - out, _ := sess.StdoutPipe() - - err = sess.Start(strings.TrimSpace(cmd)) - if err != nil { - exitCode = -1 - if exitErr, ok := err.(*ssh.ExitError); ok { - exitCode = exitErr.ExitStatus() - } - return "", exitCode, err - } - - var ( - output []byte - line = "" - r = bufio.NewReader(out) - ) - - for { - b, err := r.ReadByte() - if err != nil { - break - } - - output = append(output, b) - - if b == byte('\n') { - line = "" - continue - } - - line += string(b) - - if (strings.HasPrefix(line, "[sudo] password for ") || strings.HasPrefix(line, "Password")) && strings.HasSuffix(line, ": ") { - _, err = in.Write([]byte(host.GetPassword() + "\n")) - if err != nil { - break - } - } - } - err = sess.Wait() - if err != nil { - exitCode = -1 - if exitErr, ok := err.(*ssh.ExitError); ok { - exitCode = exitErr.ExitStatus() - } - } - outStr := strings.TrimPrefix(string(output), fmt.Sprintf("[sudo] password for %s:", host.GetUser())) - - // preserve original error - return strings.TrimSpace(outStr), exitCode, errors.Wrapf(err, "Failed to exec command: %s \n%s", cmd, strings.TrimSpace(outStr)) -} - -func (c *connection) Fetch(local, remote string, host Host) error { - //srcFile, err := c.sftpclient.Open(remote) - //if err != nil { - // return fmt.Errorf("open remote file failed %v, remote path: %s", err, remote) - //} - //defer srcFile.Close() - - // Base64 encoding is performed on the contents of the file to prevent garbled code in the target file. - output, _, err := c.Exec(SudoPrefix(fmt.Sprintf("cat %s | base64 -w 0", remote)), host) - if err != nil { - return fmt.Errorf("open remote file failed %v, remote path: %s", err, remote) - } - - err = util.MkFileFullPathDir(local) - if err != nil { - return err - } - // open local Destination file - dstFile, err := os.Create(local) - if err != nil { - return fmt.Errorf("create local file failed %v", err) - } - defer dstFile.Close() - // copy to local file - //_, err = srcFile.WriteTo(dstFile) - if base64Str, err := base64.StdEncoding.DecodeString(output); err != nil { - return err - } else { - if _, err = dstFile.WriteString(string(base64Str)); err != nil { - return err - } - } - - return nil -} - -type scpErr struct { - err error -} - -func (c *connection) Scp(src, dst string, host Host) error { - baseRemotePath := filepath.Dir(dst) - - if err := c.MkDirAll(baseRemotePath, "777", host); err != nil { - return err - } - f, err := os.Stat(src) - if err != nil { - return fmt.Errorf("get file stat failed: %s", err) - } - - number := 1 - if f.IsDir() { - number = util.CountDirFiles(src) - } - // empty dir - if number == 0 { - return nil - } - - scpErr := new(scpErr) - if f.IsDir() { - c.copyDirToRemote(src, dst, scpErr, host) - if scpErr.err != nil { - return scpErr.err - } - } else { - if err := c.copyFileToRemote(src, dst, host); err != nil { - return err - } - } - return nil -} - -func (c *connection) copyDirToRemote(src, dst string, scrErr *scpErr, host Host) { - localFiles, err := os.ReadDir(src) - if err != nil { - logger.Log.Errorf("read local path dir %s failed %v", src, err) - scrErr.err = err - return - } - if err = c.MkDirAll(dst, "", host); err != nil { - logger.Log.Errorf("failed to create remote path %s:%v", dst, err) - scrErr.err = err - return - } - for _, file := range localFiles { - local := path.Join(src, file.Name()) - remote := path.Join(dst, file.Name()) - if file.IsDir() { - if err = c.MkDirAll(remote, "", host); err != nil { - logger.Log.Errorf("failed to create remote path %s:%v", remote, err) - scrErr.err = err - return - } - c.copyDirToRemote(local, remote, scrErr, host) - } else { - err := c.copyFileToRemote(local, remote, host) - if err != nil { - logger.Log.Errorf("copy local file %s to remote file %s failed %v ", local, remote, err) - scrErr.err = err - return - } - } - } -} - -func (c *connection) copyFileToRemote(src, dst string, host Host) error { - // check remote file md5 first - var ( - srcMd5, dstMd5 string - ) - srcMd5 = util.LocalMd5Sum(src) - if c.RemoteFileExist(dst, host) { - dstMd5 = c.RemoteMd5Sum(dst, host) - if srcMd5 == dstMd5 { - logger.Log.Debug("remote file %s md5 value is the same as local file, skip scp", dst) - return nil - } - } - - srcFile, err := os.Open(src) - if err != nil { - return err - } - defer srcFile.Close() - // the dst file mod will be 0666 - dstFile, err := c.sftpclient.Create(dst) - if err != nil { - return err - } - fileStat, err := srcFile.Stat() - if err != nil { - return fmt.Errorf("get file stat failed %v", err) - } - if err := dstFile.Chmod(fileStat.Mode()); err != nil { - return fmt.Errorf("chmod remote file failed %v", err) - } - defer dstFile.Close() - _, err = io.Copy(dstFile, srcFile) - if err != nil { - return err - } - dstMd5 = c.RemoteMd5Sum(dst, host) - if srcMd5 != dstMd5 { - return fmt.Errorf("validate md5sum failed %s != %s", srcMd5, dstMd5) - } - return nil -} - -func (c *connection) RemoteMd5Sum(dst string, host Host) string { - cmd := fmt.Sprintf("md5sum %s | cut -d\" \" -f1", dst) - remoteMd5, _, err := c.Exec(cmd, host) - if err != nil { - logger.Log.Errorf("exec countRemoteMd5Command %s failed: %v", cmd, err) - } - return remoteMd5 -} - -func (c *connection) RemoteFileExist(dst string, host Host) bool { - remoteFileName := path.Base(dst) - remoteFileDirName := path.Dir(dst) - - remoteFileCommand := fmt.Sprintf(SudoPrefix("ls -l %s/%s 2>/dev/null |wc -l"), remoteFileDirName, remoteFileName) - - out, _, err := c.Exec(remoteFileCommand, host) - defer func() { - if r := recover(); r != nil { - logger.Log.Errorf("exec remoteFileCommand %s err: %v", remoteFileCommand, err) - } - }() - if err != nil { - panic(1) - } - count, err := strconv.Atoi(strings.TrimSpace(out)) - defer func() { - if r := recover(); r != nil { - logger.Log.Errorf("check remote file exist err: %v", err) - } - }() - if err != nil { - panic(1) - } - return count != 0 -} - -func (c *connection) RemoteDirExist(dst string, host Host) (bool, error) { - if _, err := c.sftpclient.ReadDir(dst); err != nil { - return false, err - } - return true, nil -} - -func (c *connection) MkDirAll(path string, mode string, host Host) error { - if mode == "" { - mode = "775" - } - mkDstDir := fmt.Sprintf("mkdir -p -m %s %s || true", mode, path) - if strings.Contains(path, common.TmpDir) { - mkDstDir = fmt.Sprintf("mkdir -p %s && chmod -R %s %s || true", path, mode, common.TmpDir) - } - if _, _, err := c.Exec(SudoPrefix(mkDstDir), host); err != nil { - return err - } - - return nil -} - -func (c *connection) Chmod(path string, mode os.FileMode) error { - remotePath := filepath.Dir(path) - if err := c.sftpclient.Chmod(remotePath, mode); err != nil { - return err - } - return nil -} - -func SudoPrefix(cmd string) string { - return fmt.Sprintf("sudo -E /bin/bash -c \"%s\"", cmd) -} diff --git a/cmd/kk/pkg/core/connector/tee.go b/cmd/kk/pkg/core/connector/tee.go deleted file mode 100644 index 2ba468e7c..000000000 --- a/cmd/kk/pkg/core/connector/tee.go +++ /dev/null @@ -1,48 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package connector - -import ( - "bytes" - "io" - "strings" -) - -type Tee struct { - buffer bytes.Buffer - upstream io.WriteCloser -} - -// NewTee constructor -func NewTee(wc io.WriteCloser) *Tee { - return &Tee{upstream: wc} -} - -func (t *Tee) Write(p []byte) (int, error) { - t.buffer.Write(p) - - return t.upstream.Write(p) -} - -func (t *Tee) String() string { - return strings.TrimSpace(t.buffer.String()) -} - -// Close underlying io.Closer -func (t *Tee) Close() error { - return t.upstream.Close() -} diff --git a/cmd/kk/pkg/core/ending/action_result.go b/cmd/kk/pkg/core/ending/action_result.go deleted file mode 100644 index 7f26d92ac..000000000 --- a/cmd/kk/pkg/core/ending/action_result.go +++ /dev/null @@ -1,51 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package ending - -import ( - "time" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type ActionResult struct { - Host connector.Host - Status ResultStatus - Error error - StartTime time.Time - EndTime time.Time -} - -func (a *ActionResult) GetHost() connector.Host { - return a.Host -} - -func (a *ActionResult) GetStatus() ResultStatus { - return a.Status -} - -func (a *ActionResult) GetErr() error { - return a.Error -} - -func (a *ActionResult) GetStartTime() time.Time { - return a.StartTime -} - -func (a *ActionResult) GetEndTime() time.Time { - return a.EndTime -} diff --git a/cmd/kk/pkg/core/ending/interface.go b/cmd/kk/pkg/core/ending/interface.go deleted file mode 100644 index ee55753b3..000000000 --- a/cmd/kk/pkg/core/ending/interface.go +++ /dev/null @@ -1,31 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package ending - -import ( - "time" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type Interface interface { - GetHost() connector.Host - GetStatus() ResultStatus - GetErr() error - GetStartTime() time.Time - GetEndTime() time.Time -} diff --git a/cmd/kk/pkg/core/ending/module_result.go b/cmd/kk/pkg/core/ending/module_result.go deleted file mode 100644 index 88d2757cd..000000000 --- a/cmd/kk/pkg/core/ending/module_result.go +++ /dev/null @@ -1,74 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package ending - -import ( - "time" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type ModuleResult struct { - HostResults map[string]Interface - CombineResult error - Status ResultStatus - StartTime time.Time - EndTime time.Time -} - -func NewModuleResult() *ModuleResult { - return &ModuleResult{HostResults: make(map[string]Interface), StartTime: time.Now(), Status: NULL} -} - -func (m *ModuleResult) IsFailed() bool { - return m.Status == FAILED -} - -func (m *ModuleResult) AppendHostResult(p Interface) { - if m.HostResults == nil { - return - } - m.HostResults[p.GetHost().GetName()] = p -} - -func (m *ModuleResult) LocalErrResult(err error) { - now := time.Now() - r := &ActionResult{ - Host: &connector.BaseHost{Name: common.LocalHost}, - Status: FAILED, - Error: err, - StartTime: m.StartTime, - EndTime: now, - } - - m.HostResults[r.Host.GetName()] = r - m.CombineResult = err - m.EndTime = now - m.Status = FAILED -} - -func (m *ModuleResult) ErrResult(combineErr error) { - m.EndTime = time.Now() - m.Status = FAILED - m.CombineResult = combineErr -} - -func (m *ModuleResult) NormalResult() { - m.EndTime = time.Now() - m.Status = SUCCESS -} diff --git a/cmd/kk/pkg/core/ending/status_enum.go b/cmd/kk/pkg/core/ending/status_enum.go deleted file mode 100644 index aa8779d1f..000000000 --- a/cmd/kk/pkg/core/ending/status_enum.go +++ /dev/null @@ -1,61 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package ending - -type ResultStatus int - -const ( - NULL ResultStatus = -99 - SKIPPED ResultStatus = iota - 2 - SUCCESS - FAILED -) - -var EnumList = []ResultStatus{ - NULL, - SKIPPED, - SUCCESS, - FAILED, -} - -func (r ResultStatus) String() string { - switch r { - case SUCCESS: - return "success" - case FAILED: - return "failed" - case SKIPPED: - return "skipped" - case NULL: - return "null" - default: - return "invalid option" - } -} - -func GetByCode(code int) ResultStatus { - switch code { - case -99: - return NULL - case -1: - return SKIPPED - case 0: - return SUCCESS - default: - return FAILED - } -} diff --git a/cmd/kk/pkg/core/ending/task_result.go b/cmd/kk/pkg/core/ending/task_result.go deleted file mode 100644 index 73565e95a..000000000 --- a/cmd/kk/pkg/core/ending/task_result.go +++ /dev/null @@ -1,134 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package ending - -import ( - "fmt" - "sync" - "time" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type TaskResult struct { - mu sync.Mutex - ActionResults []*ActionResult - Status ResultStatus - StartTime time.Time - EndTime time.Time -} - -func NewTaskResult() *TaskResult { - return &TaskResult{ActionResults: make([]*ActionResult, 0, 0), Status: NULL, StartTime: time.Now()} -} - -func (t *TaskResult) ErrResult() { - if t.Status != NULL { - return - } - t.EndTime = time.Now() - t.Status = FAILED -} - -func (t *TaskResult) NormalResult() { - if t.Status != NULL { - return - } - t.EndTime = time.Now() - t.Status = SUCCESS -} - -func (t *TaskResult) SkippedResult() { - if t.Status != NULL { - return - } - t.EndTime = time.Now() - t.Status = SKIPPED -} - -func (t *TaskResult) AppendSkip(host connector.Host) { - t.mu.Lock() - defer t.mu.Unlock() - now := time.Now() - e := &ActionResult{ - Host: host, - Status: SKIPPED, - Error: nil, - StartTime: t.StartTime, - EndTime: now, - } - - t.ActionResults = append(t.ActionResults, e) - t.EndTime = now -} - -func (t *TaskResult) AppendSuccess(host connector.Host) { - t.mu.Lock() - defer t.mu.Unlock() - now := time.Now() - e := &ActionResult{ - Host: host, - Status: SUCCESS, - Error: nil, - StartTime: t.StartTime, - EndTime: now, - } - - t.ActionResults = append(t.ActionResults, e) - t.EndTime = now -} - -func (t *TaskResult) AppendErr(host connector.Host, err error) { - t.mu.Lock() - defer t.mu.Unlock() - now := time.Now() - e := &ActionResult{ - Host: host, - Status: FAILED, - Error: err, - StartTime: t.StartTime, - EndTime: now, - } - - t.ActionResults = append(t.ActionResults, e) - t.EndTime = now - t.Status = FAILED -} - -func (t *TaskResult) IsFailed() bool { - t.mu.Lock() - defer t.mu.Unlock() - return t.Status == FAILED -} - -func (t *TaskResult) CombineErr() error { - t.mu.Lock() - defer t.mu.Unlock() - if len(t.ActionResults) != 0 { - var str string - for i := range t.ActionResults { - if t.ActionResults[i].Status != FAILED { - continue - } - str += fmt.Sprintf("\nfailed: [%s] %s", t.ActionResults[i].Host.GetName(), t.ActionResults[i].Error.Error()) - } - return errors.New(str) - } - return nil -} diff --git a/cmd/kk/pkg/core/hook/handler.go b/cmd/kk/pkg/core/hook/handler.go deleted file mode 100644 index f5c61b13e..000000000 --- a/cmd/kk/pkg/core/hook/handler.go +++ /dev/null @@ -1,23 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package hook - -func Call(hook Interface) error { - err := hook.Catch(hook.Try()) - hook.Finally() - return err -} diff --git a/cmd/kk/pkg/core/hook/interface.go b/cmd/kk/pkg/core/hook/interface.go deleted file mode 100644 index 4715f4bcd..000000000 --- a/cmd/kk/pkg/core/hook/interface.go +++ /dev/null @@ -1,23 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package hook - -type Interface interface { - Try() error - Catch(err error) error - Finally() -} diff --git a/cmd/kk/pkg/core/logger/formatter.go b/cmd/kk/pkg/core/logger/formatter.go deleted file mode 100644 index 748d27854..000000000 --- a/cmd/kk/pkg/core/logger/formatter.go +++ /dev/null @@ -1,204 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package logger - -import ( - "bytes" - "fmt" - "github.com/sirupsen/logrus" - "runtime" - "sort" - "strings" - "time" -) - -type Formatter struct { - // TimestampFormat - default: time.StampMilli = "Jan _2 15:04:05.000" - TimestampFormat string - // NoColors - disable colors - NoColors bool - // ShowLevel - when the level < this field, it won't be show. default: TRACE - ShowLevel logrus.Level - // ShowFullLevel - show a full level [WARNING] instead of [WARN] - ShowFullLevel bool - // NoUppercaseLevel - no upper case for level value - NoUppercaseLevel bool - // HideKeys - show [fieldValue] instead of [fieldKey:fieldValue] - HideKeys bool - // FieldsDisplayWithOrder - default: all fields display and sorted alphabetically - FieldsDisplayWithOrder []string - // CallerFirst - print caller info first - CallerFirst bool - // CustomCallerFormatter - set custom formatter for caller info - CustomCallerFormatter func(*runtime.Frame) string -} - -func (f *Formatter) Format(entry *logrus.Entry) ([]byte, error) { - levelColor := getColorByLevel(entry.Level) - - timestampFormat := f.TimestampFormat - if timestampFormat == "" { - timestampFormat = time.StampMilli - } - - // output buffer - b := &bytes.Buffer{} - - // write time - b.WriteString(entry.Time.Format(timestampFormat)) - - if f.CallerFirst { - f.writeCaller(b, entry) - } - - if !f.NoColors { - fmt.Fprintf(b, "\x1b[%dm", levelColor) - } - - level := entry.Level - if f.ShowLevel >= level { - var levelStr string - if f.NoUppercaseLevel { - levelStr = entry.Level.String() - } else { - levelStr = strings.ToUpper(entry.Level.String()) - } - - b.WriteString(" [") - if f.ShowFullLevel { - b.WriteString(levelStr) - } else { - b.WriteString(levelStr[:4]) - } - b.WriteString("]") - } - - b.WriteString(" ") - - // write fields - if f.FieldsDisplayWithOrder == nil { - f.writeFields(b, entry) - } else { - f.writeOrderedFields(b, entry) - } - - b.WriteString(entry.Message) - - if !f.CallerFirst { - f.writeCaller(b, entry) - } - - b.WriteByte('\n') - - return b.Bytes(), nil -} - -const ( - colorRed = 31 - colorYellow = 33 - colorBlue = 36 - colorGray = 37 -) - -func getColorByLevel(level logrus.Level) int { - switch level { - case logrus.DebugLevel, logrus.TraceLevel: - return colorGray - case logrus.WarnLevel: - return colorYellow - case logrus.ErrorLevel, logrus.FatalLevel, logrus.PanicLevel: - return colorRed - default: - return colorBlue - } -} - -func (f *Formatter) writeFields(b *bytes.Buffer, entry *logrus.Entry) { - if len(entry.Data) != 0 { - fields := make([]string, 0, len(entry.Data)) - for field := range entry.Data { - fields = append(fields, field) - } - - sort.Strings(fields) - - b.WriteString("[") - for i, field := range fields { - f.writeField(b, entry, field, i) - } - b.WriteString("]") - } -} - -func (f *Formatter) writeOrderedFields(b *bytes.Buffer, entry *logrus.Entry) { - if len(entry.Data) != 0 { - b.WriteString("[") - length := len(entry.Data) - foundFieldsMap := map[string]bool{} - for i, field := range f.FieldsDisplayWithOrder { - if _, ok := entry.Data[field]; ok { - foundFieldsMap[field] = true - length-- - f.writeField(b, entry, field, i) - } - } - - if length > 0 { - notFoundFields := make([]string, 0, length) - for field := range entry.Data { - if foundFieldsMap[field] == false { - notFoundFields = append(notFoundFields, field) - } - } - - sort.Strings(notFoundFields) - - for i, field := range notFoundFields { - f.writeField(b, entry, field, i) - } - } - b.WriteString("]") - } -} - -func (f *Formatter) writeField(b *bytes.Buffer, entry *logrus.Entry, field string, i int) { - if f.HideKeys { - fmt.Fprintf(b, "%v", entry.Data[field]) - } else { - fmt.Fprintf(b, "%s:%v", field, entry.Data[field]) - } - - if i != len(entry.Data) && len(entry.Data) != 1 { - b.WriteString(" | ") - } -} - -func (f *Formatter) writeCaller(b *bytes.Buffer, entry *logrus.Entry) { - if entry.HasCaller() { - if f.CustomCallerFormatter != nil { - fmt.Fprintf(b, f.CustomCallerFormatter(entry.Caller)) - } else { - fmt.Fprintf( - b, - " (%s:%d %s)", - entry.Caller.File, - entry.Caller.Line, - entry.Caller.Function, - ) - } - } -} diff --git a/cmd/kk/pkg/core/logger/logger.go b/cmd/kk/pkg/core/logger/logger.go deleted file mode 100644 index ad3b2295d..000000000 --- a/cmd/kk/pkg/core/logger/logger.go +++ /dev/null @@ -1,83 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package logger - -import ( - "fmt" - "path/filepath" - "time" - - rotatelogs "github.com/lestrrat-go/file-rotatelogs" - "github.com/rifflock/lfshook" - "github.com/sirupsen/logrus" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/common" -) - -var Log *KubeKeyLog - -type KubeKeyLog struct { - logrus.FieldLogger - OutputPath string - Verbose bool -} - -func NewLogger(outputPath string, verbose bool) *KubeKeyLog { - logger := logrus.New() - - formatter := &Formatter{ - HideKeys: true, - TimestampFormat: "15:04:05 MST", - NoColors: true, - ShowLevel: logrus.WarnLevel, - FieldsDisplayWithOrder: []string{common.Pipeline, common.Module, common.Task, common.Node}, - } - logger.SetFormatter(formatter) - - path := filepath.Join(outputPath, "./kubekey.log") - writer, _ := rotatelogs.New( - path+".%Y%m%d", - rotatelogs.WithLinkName(path), - rotatelogs.WithRotationTime(24*time.Hour), - ) - - logWriters := lfshook.WriterMap{ - logrus.InfoLevel: writer, - logrus.WarnLevel: writer, - logrus.ErrorLevel: writer, - logrus.FatalLevel: writer, - logrus.PanicLevel: writer, - } - - if verbose { - logger.SetLevel(logrus.DebugLevel) - logWriters[logrus.DebugLevel] = writer - } else { - logger.SetLevel(logrus.InfoLevel) - } - - logger.Hooks.Add(lfshook.NewHook(logWriters, formatter)) - return &KubeKeyLog{logger, outputPath, verbose} -} - -func (k *KubeKeyLog) Message(node, str string) { - Log.Infof("message: [%s]\n%s", node, str) -} - -func (k *KubeKeyLog) Messagef(node, format string, args ...interface{}) { - Log.Infof("message: [%s]\n%s", node, fmt.Sprintf(format, args...)) -} diff --git a/cmd/kk/pkg/core/logger/logger_test.go b/cmd/kk/pkg/core/logger/logger_test.go deleted file mode 100644 index 1370a1353..000000000 --- a/cmd/kk/pkg/core/logger/logger_test.go +++ /dev/null @@ -1,42 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package logger - -import ( - "sync" - "testing" -) - -var log = NewLogger("", true) - -func TestKubeKey_Print(t *testing.T) { - wg := &sync.WaitGroup{} - for i := 0; i < 5; i++ { - - log.Info("empty fields") - l1 := *log - for j := 0; j < 10; j++ { - wg.Add(1) - go func(x int, log1 KubeKeyLog) { - log.Info("Congratulations!", "ssssss") - wg.Done() - }(j, l1) - } - wg.Wait() - } - -} diff --git a/cmd/kk/pkg/core/module/base.go b/cmd/kk/pkg/core/module/base.go deleted file mode 100644 index 7bfc84a81..000000000 --- a/cmd/kk/pkg/core/module/base.go +++ /dev/null @@ -1,85 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package module - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/ending" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/hook" -) - -type BaseModule struct { - Name string - Desc string - Skip bool - ModuleCache *cache.Cache - PipelineCache *cache.Cache - Runtime connector.ModuleRuntime - PostHook []PostHookInterface -} - -func (b *BaseModule) IsSkip() bool { - return b.Skip -} - -func (b *BaseModule) Default(runtime connector.Runtime, pipelineCache *cache.Cache, moduleCache *cache.Cache) { - b.Runtime = runtime - b.PipelineCache = pipelineCache - b.ModuleCache = moduleCache -} - -func (b *BaseModule) Init() { - if b.Name == "" { - b.Name = DefaultModuleName - } -} - -func (b *BaseModule) Is() string { - return BaseModuleType -} - -func (b *BaseModule) Run(result *ending.ModuleResult) { - panic("implement me") -} - -func (b *BaseModule) Until() (*bool, error) { - return nil, nil -} - -func (b *BaseModule) Slogan() { -} - -func (b *BaseModule) AutoAssert() { -} - -func (b *BaseModule) AppendPostHook(h PostHookInterface) { - b.PostHook = append(b.PostHook, h) -} - -func (b *BaseModule) CallPostHook(result *ending.ModuleResult) error { - for i := range b.PostHook { - h := b.PostHook[i] - h.Init(b, result) - if err := hook.Call(h); err != nil { - return errors.Wrapf(err, "Module[%s] call post hook failed", b.Name) - } - } - return nil -} diff --git a/cmd/kk/pkg/core/module/const.go b/cmd/kk/pkg/core/module/const.go deleted file mode 100644 index 65e7b1f2b..000000000 --- a/cmd/kk/pkg/core/module/const.go +++ /dev/null @@ -1,29 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package module - -const ( - DefaultModuleName = "DefaultBaseModule" - DefaultTaskModuleName = "DefaultTaskModule" - DefaultGoroutineModuleName = "DefaultGoroutineModuleName" - DefaultCustomModuleName = "DefaultCustomModule" - - BaseModuleType = "BaseModuleType" - TaskModuleType = "TaskModuleType" - GoroutineModuleType = "GoroutineModuleType" - CustomModuleType = "CustomModuleType" -) diff --git a/cmd/kk/pkg/core/module/custom_module.go b/cmd/kk/pkg/core/module/custom_module.go deleted file mode 100644 index 586644abf..000000000 --- a/cmd/kk/pkg/core/module/custom_module.go +++ /dev/null @@ -1,31 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package module - -type CustomModule struct { - BaseModule -} - -func (c *CustomModule) Init() { - if c.Name == "" { - c.Name = DefaultCustomModuleName - } -} - -func (c *CustomModule) Is() string { - return CustomModuleType -} diff --git a/cmd/kk/pkg/core/module/hook.go b/cmd/kk/pkg/core/module/hook.go deleted file mode 100644 index 58a450407..000000000 --- a/cmd/kk/pkg/core/module/hook.go +++ /dev/null @@ -1,48 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package module - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/ending" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/hook" -) - -type PostHookInterface interface { - hook.Interface - Init(module Module, result *ending.ModuleResult) -} - -type PostHook struct { - Module Module - Result *ending.ModuleResult -} - -func (p *PostHook) Try() error { - panic("implement me") -} - -func (p *PostHook) Catch(err error) error { - return err -} - -func (p *PostHook) Finally() { -} - -func (p *PostHook) Init(module Module, result *ending.ModuleResult) { - p.Module = module - p.Result = result -} diff --git a/cmd/kk/pkg/core/module/interface.go b/cmd/kk/pkg/core/module/interface.go deleted file mode 100644 index 36a07334f..000000000 --- a/cmd/kk/pkg/core/module/interface.go +++ /dev/null @@ -1,36 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package module - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/ending" -) - -type Module interface { - IsSkip() bool - Default(runtime connector.Runtime, pipelineCache *cache.Cache, moduleCache *cache.Cache) - Init() - Is() string - Run(result *ending.ModuleResult) - Until() (*bool, error) - Slogan() - AutoAssert() - AppendPostHook(h PostHookInterface) - CallPostHook(result *ending.ModuleResult) error -} diff --git a/cmd/kk/pkg/core/module/task_module.go b/cmd/kk/pkg/core/module/task_module.go deleted file mode 100644 index 569302e0e..000000000 --- a/cmd/kk/pkg/core/module/task_module.go +++ /dev/null @@ -1,77 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package module - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/ending" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" -) - -type BaseTaskModule struct { - BaseModule - Tasks []task.Interface -} - -func (b *BaseTaskModule) Init() { - if b.Name == "" { - b.Name = DefaultTaskModuleName - } -} - -func (b *BaseTaskModule) Is() string { - return TaskModuleType -} - -func (b *BaseTaskModule) Run(result *ending.ModuleResult) { - for i := range b.Tasks { - t := b.Tasks[i] - t.Init(b.Runtime.(connector.Runtime), b.ModuleCache, b.PipelineCache) - - logger.Log.Infof("[%s] %s", b.Name, t.GetDesc()) - res := t.Execute() - for j := range res.ActionResults { - ac := res.ActionResults[j] - logger.Log.Infof("%s: [%s]", ac.Status.String(), ac.Host.GetName()) - result.AppendHostResult(ac) - - if _, ok := t.(*task.RemoteTask); ok { - if b.Runtime.GetIgnoreErr() { - if len(b.Runtime.GetAllHosts()) > 0 { - if ac.GetStatus() == ending.FAILED { - res.Status = ending.SUCCESS - b.Runtime.DeleteHost(ac.Host) - } - } else { - result.ErrResult(errors.Wrapf(res.CombineErr(), "Module[%s] exec failed", b.Name)) - return - } - } - } - } - - if res.IsFailed() { - t.ExecuteRollback() - result.ErrResult(errors.Wrapf(res.CombineErr(), "Module[%s] exec failed", b.Name)) - return - } - } - result.NormalResult() -} diff --git a/cmd/kk/pkg/core/pipeline/pipeline.go b/cmd/kk/pkg/core/pipeline/pipeline.go deleted file mode 100644 index 9afce39d9..000000000 --- a/cmd/kk/pkg/core/pipeline/pipeline.go +++ /dev/null @@ -1,164 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pipeline - -import ( - "fmt" - "os" - "sync" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/ending" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" -) - -var logo = ` - - _ __ _ _ __ -| | / / | | | | / / -| |/ / _ _| |__ ___| |/ / ___ _ _ -| \| | | | '_ \ / _ \ \ / _ \ | | | -| |\ \ |_| | |_) | __/ |\ \ __/ |_| | -\_| \_/\__,_|_.__/ \___\_| \_/\___|\__, | - __/ | - |___/ - -` - -type Pipeline struct { - Name string - Modules []module.Module - Runtime connector.Runtime - SpecHosts int - PipelineCache *cache.Cache - ModuleCachePool sync.Pool - ModulePostHooks []module.PostHookInterface -} - -func (p *Pipeline) Init() error { - fmt.Print(logo) - p.PipelineCache = cache.NewCache() - p.SpecHosts = len(p.Runtime.GetAllHosts()) - //if err := p.Runtime.GenerateWorkDir(); err != nil { - // return err - //} - //if err := p.Runtime.InitLogger(); err != nil { - // return err - //} - return nil -} - -func (p *Pipeline) Start() error { - if err := p.Init(); err != nil { - return errors.Wrapf(err, "Pipeline[%s] execute failed", p.Name) - } - for i := range p.Modules { - m := p.Modules[i] - if m.IsSkip() { - continue - } - - moduleCache := p.newModuleCache() - m.Default(p.Runtime, p.PipelineCache, moduleCache) - m.AutoAssert() - m.Init() - for j := range p.ModulePostHooks { - m.AppendPostHook(p.ModulePostHooks[j]) - } - - res := p.RunModule(m) - err := m.CallPostHook(res) - if res.IsFailed() { - return errors.Wrapf(res.CombineResult, "Pipeline[%s] execute failed", p.Name) - } - if err != nil { - return errors.Wrapf(err, "Pipeline[%s] execute failed", p.Name) - } - p.releaseModuleCache(moduleCache) - } - p.releasePipelineCache() - - // close ssh connect - for _, host := range p.Runtime.GetAllHosts() { - p.Runtime.GetConnector().Close(host) - } - - if p.SpecHosts != len(p.Runtime.GetAllHosts()) { - return errors.Errorf("Pipeline[%s] execute failed: there are some error in your spec hosts", p.Name) - } - logger.Log.Infof("Pipeline[%s] execute successfully", p.Name) - return nil -} - -func (p *Pipeline) RunModule(m module.Module) *ending.ModuleResult { - m.Slogan() - - result := ending.NewModuleResult() - for { - switch m.Is() { - case module.TaskModuleType: - m.Run(result) - if result.IsFailed() { - return result - } - - case module.GoroutineModuleType: - go func() { - m.Run(result) - if result.IsFailed() { - os.Exit(1) - } - }() - default: - m.Run(result) - if result.IsFailed() { - return result - } - } - - stop, err := m.Until() - if err != nil { - result.LocalErrResult(err) - return result - } - if stop == nil || *stop { - break - } - } - return result -} - -func (p *Pipeline) newModuleCache() *cache.Cache { - moduleCache, ok := p.ModuleCachePool.Get().(*cache.Cache) - if ok { - return moduleCache - } - return cache.NewCache() -} - -func (p *Pipeline) releasePipelineCache() { - p.PipelineCache.Clean() -} - -func (p *Pipeline) releaseModuleCache(c *cache.Cache) { - c.Clean() - p.ModuleCachePool.Put(c) -} diff --git a/cmd/kk/pkg/core/prepare/base.go b/cmd/kk/pkg/core/prepare/base.go deleted file mode 100644 index 46b644b09..000000000 --- a/cmd/kk/pkg/core/prepare/base.go +++ /dev/null @@ -1,66 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package prepare - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type BasePrepare struct { - ModuleCache *cache.Cache - PipelineCache *cache.Cache -} - -func (b *BasePrepare) Init(moduleCache *cache.Cache, pipelineCache *cache.Cache) { - b.ModuleCache = moduleCache - b.PipelineCache = pipelineCache -} - -func (b *BasePrepare) PreCheck(runtime connector.Runtime) (bool, error) { - return true, nil -} - -func (b *BasePrepare) AutoAssert(runtime connector.Runtime) { -} - -type PrepareCollection []Prepare - -func (p *PrepareCollection) Init(cache *cache.Cache, rootCache *cache.Cache) { - for _, v := range *p { - v.Init(cache, rootCache) - } -} - -func (p *PrepareCollection) PreCheck(runtime connector.Runtime) (bool, error) { - for _, v := range *p { - res, err := v.PreCheck(runtime) - if err != nil { - return false, err - } - if res == false { - return false, nil - } - } - return true, nil -} - -func (p *PrepareCollection) AutoAssert(runtime connector.Runtime) { - for _, v := range *p { - v.AutoAssert(runtime) - } -} diff --git a/cmd/kk/pkg/core/prepare/fast_prepare.go b/cmd/kk/pkg/core/prepare/fast_prepare.go deleted file mode 100644 index 7f1566b84..000000000 --- a/cmd/kk/pkg/core/prepare/fast_prepare.go +++ /dev/null @@ -1,30 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package prepare - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type FastPrepare struct { - BasePrepare - Inject func(runtime connector.Runtime) (bool, error) -} - -func (b *FastPrepare) PreCheck(runtime connector.Runtime) (bool, error) { - return b.Inject(runtime) -} diff --git a/cmd/kk/pkg/core/prepare/file_check.go b/cmd/kk/pkg/core/prepare/file_check.go deleted file mode 100644 index b0cc21edf..000000000 --- a/cmd/kk/pkg/core/prepare/file_check.go +++ /dev/null @@ -1,38 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package prepare - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type FileExist struct { - BasePrepare - FilePath string - Not bool -} - -func (f *FileExist) PreCheck(runtime connector.Runtime) (bool, error) { - exist, err := runtime.GetRunner().FileExist(f.FilePath) - if err != nil { - return false, err - } - if f.Not { - return !exist, nil - } - return exist, nil -} diff --git a/cmd/kk/pkg/core/prepare/interface.go b/cmd/kk/pkg/core/prepare/interface.go deleted file mode 100644 index 46eb02dde..000000000 --- a/cmd/kk/pkg/core/prepare/interface.go +++ /dev/null @@ -1,28 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package prepare - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type Prepare interface { - PreCheck(runtime connector.Runtime) (bool, error) - Init(cache *cache.Cache, rootCache *cache.Cache) - AutoAssert(runtime connector.Runtime) -} diff --git a/cmd/kk/pkg/core/rollback/base.go b/cmd/kk/pkg/core/rollback/base.go deleted file mode 100644 index 838cd8ba2..000000000 --- a/cmd/kk/pkg/core/rollback/base.go +++ /dev/null @@ -1,41 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package rollback - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/ending" -) - -type BaseRollback struct { - ModuleCache *cache.Cache - PipelineCache *cache.Cache -} - -func (b *BaseRollback) Init(moduleCache *cache.Cache, pipelineCache *cache.Cache) { - b.ModuleCache = moduleCache - b.PipelineCache = pipelineCache -} - -func (b *BaseRollback) Execute(runtime connector.Runtime, result *ending.ActionResult) error { - return nil -} - -func (b *BaseRollback) AutoAssert(runtime connector.Runtime) { - -} diff --git a/cmd/kk/pkg/core/rollback/interface.go b/cmd/kk/pkg/core/rollback/interface.go deleted file mode 100644 index f0a28b230..000000000 --- a/cmd/kk/pkg/core/rollback/interface.go +++ /dev/null @@ -1,29 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package rollback - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/ending" -) - -type Rollback interface { - Execute(runtime connector.Runtime, result *ending.ActionResult) (err error) - Init(cache *cache.Cache, rootCache *cache.Cache) - AutoAssert(runtime connector.Runtime) -} diff --git a/cmd/kk/pkg/core/task/const.go b/cmd/kk/pkg/core/task/const.go deleted file mode 100644 index eff13ee6e..000000000 --- a/cmd/kk/pkg/core/task/const.go +++ /dev/null @@ -1,24 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package task - -const ( - DefaultTimeout = 120 - DefaultCon = 10 - - DefaultTaskName = "DefaultTask" -) diff --git a/cmd/kk/pkg/core/task/interface.go b/cmd/kk/pkg/core/task/interface.go deleted file mode 100644 index d1d154e7f..000000000 --- a/cmd/kk/pkg/core/task/interface.go +++ /dev/null @@ -1,30 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package task - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/ending" -) - -type Interface interface { - GetDesc() string - Init(runtime connector.Runtime, moduleCache *cache.Cache, pipelineCache *cache.Cache) - Execute() *ending.TaskResult - ExecuteRollback() -} diff --git a/cmd/kk/pkg/core/task/local_task.go b/cmd/kk/pkg/core/task/local_task.go deleted file mode 100644 index 0a197c648..000000000 --- a/cmd/kk/pkg/core/task/local_task.go +++ /dev/null @@ -1,231 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package task - -import ( - "context" - "fmt" - "time" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/ending" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/rollback" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -type LocalTask struct { - Name string - Desc string - Prepare prepare.Prepare - Action action.Action - Rollback rollback.Rollback - Retry int - Delay time.Duration - Timeout time.Duration - - PipelineCache *cache.Cache - ModuleCache *cache.Cache - Runtime connector.Runtime - tag string - IgnoreError bool - TaskResult *ending.TaskResult -} - -func (l *LocalTask) GetDesc() string { - return l.Desc -} - -func (l *LocalTask) Init(runtime connector.Runtime, moduleCache *cache.Cache, pipelineCache *cache.Cache) { - l.ModuleCache = moduleCache - l.PipelineCache = pipelineCache - l.Runtime = runtime - l.Default() -} - -func (l *LocalTask) Default() { - l.TaskResult = ending.NewTaskResult() - if l.Name == "" { - l.Name = DefaultTaskName - } - - if l.Prepare == nil { - l.Prepare = new(prepare.BasePrepare) - } - - if l.Action == nil { - l.TaskResult.AppendErr(nil, errors.New("the action is nil")) - return - } - - if l.Retry <= 0 { - l.Retry = 1 - } - - if l.Delay <= 0 { - l.Delay = 5 * time.Second - } - - if l.Timeout <= 0 { - l.Timeout = DefaultTimeout * time.Minute - } -} - -func (l *LocalTask) Execute() *ending.TaskResult { - if l.TaskResult.IsFailed() { - return l.TaskResult - } - - host := &connector.BaseHost{ - Name: common.LocalHost, - } - - selfRuntime := l.Runtime.Copy() - l.RunWithTimeout(selfRuntime, host) - - if l.TaskResult.IsFailed() { - l.TaskResult.ErrResult() - return l.TaskResult - } - - l.TaskResult.NormalResult() - return l.TaskResult -} - -func (l *LocalTask) RunWithTimeout(runtime connector.Runtime, host connector.Host) { - ctx, cancel := context.WithTimeout(context.Background(), l.Timeout) - defer cancel() - - resCh := make(chan error) - - go l.Run(runtime, host, resCh) - select { - case <-ctx.Done(): - l.TaskResult.AppendErr(host, fmt.Errorf("execute task timeout, Timeout=%s", util.ShortDur(l.Timeout))) - case e := <-resCh: - if e != nil { - l.TaskResult.AppendErr(host, e) - } - } -} - -func (l *LocalTask) Run(runtime connector.Runtime, host connector.Host, resCh chan error) { - var res error - defer func() { - resCh <- res - close(resCh) - }() - - runtime.SetRunner(&connector.Runner{ - Conn: nil, - //Debug: runtime.Arg.Debug, - Host: host, - }) - - l.Prepare.Init(l.ModuleCache, l.PipelineCache) - l.Prepare.AutoAssert(runtime) - if ok, err := l.WhenWithRetry(runtime, host); !ok { - if err != nil { - res = err - return - } else { - l.TaskResult.AppendSkip(host) - return - } - } - - l.Action.Init(l.ModuleCache, l.PipelineCache) - l.Action.AutoAssert(runtime) - if err := l.ExecuteWithRetry(runtime, host); err != nil { - res = err - return - } - l.TaskResult.AppendSuccess(host) -} - -func (l *LocalTask) WhenWithRetry(runtime connector.Runtime, host connector.Host) (bool, error) { - pass := false - err := fmt.Errorf("pre-check exec failed after %d retries", l.Retry) - for i := 0; i < l.Retry; i++ { - if res, e := l.When(runtime); e != nil { - logger.Log.Messagef(host.GetName(), e.Error()) - - if i == l.Retry-1 { - err = errors.New(err.Error() + e.Error()) - continue - } - logger.Log.Infof("retry: [%s]", host.GetName()) - time.Sleep(l.Delay) - continue - } else { - err = nil - pass = res - break - } - } - - return pass, err -} - -func (l *LocalTask) When(runtime connector.Runtime) (bool, error) { - if l.Prepare == nil { - return true, nil - } - if ok, err := l.Prepare.PreCheck(runtime); err != nil { - return false, err - } else if !ok { - return false, nil - } - return true, nil -} - -func (l *LocalTask) ExecuteWithRetry(runtime connector.Runtime, host connector.Host) error { - err := fmt.Errorf("[%s] exec failed after %d retries: ", l.Name, l.Retry) - for i := 0; i < l.Retry; i++ { - e := l.Action.Execute(runtime) - if e != nil { - logger.Log.Messagef(host.GetName(), e.Error()) - - if i == l.Retry-1 { - err = errors.New(err.Error() + e.Error()) - continue - } - logger.Log.Infof("retry: [%s]", host.GetName()) - time.Sleep(l.Delay) - continue - } else { - err = nil - break - } - } - return err -} - -func (l *LocalTask) ExecuteRollback() { - if l.Rollback == nil { - return - } - if !l.TaskResult.IsFailed() { - return - } -} diff --git a/cmd/kk/pkg/core/task/remote_task.go b/cmd/kk/pkg/core/task/remote_task.go deleted file mode 100644 index 4bb8c274d..000000000 --- a/cmd/kk/pkg/core/task/remote_task.go +++ /dev/null @@ -1,347 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package task - -import ( - "context" - "fmt" - "sync" - "time" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/ending" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/rollback" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -type RemoteTask struct { - Name string - Desc string - Hosts []connector.Host - Prepare prepare.Prepare - Action action.Action - Rollback rollback.Rollback - Parallel bool - Retry int - Delay time.Duration - Timeout time.Duration - Concurrency float64 - - PipelineCache *cache.Cache - ModuleCache *cache.Cache - Runtime connector.Runtime - IgnoreError bool - TaskResult *ending.TaskResult -} - -func (t *RemoteTask) GetDesc() string { - return t.Desc -} - -func (t *RemoteTask) Init(runtime connector.Runtime, moduleCache *cache.Cache, pipelineCache *cache.Cache) { - t.ModuleCache = moduleCache - t.PipelineCache = pipelineCache - t.Runtime = runtime - t.Default() -} - -func (t *RemoteTask) Execute() *ending.TaskResult { - if t.TaskResult.IsFailed() { - return t.TaskResult - } - - routinePool := make(chan struct{}, DefaultCon) - defer close(routinePool) - - ctx, cancel := context.WithTimeout(context.Background(), t.Timeout) - defer cancel() - wg := &sync.WaitGroup{} - for i := range t.Hosts { - if t.Hosts[i] == nil || t.Runtime.HostIsDeprecated(t.Hosts[i]) { - continue - } - selfRuntime := t.Runtime.Copy() - - wg.Add(1) - if t.Parallel { - go t.RunWithTimeout(ctx, selfRuntime, t.Hosts[i], i, wg, routinePool) - } else { - t.RunWithTimeout(ctx, selfRuntime, t.Hosts[i], i, wg, routinePool) - } - } - wg.Wait() - - if t.TaskResult.IsFailed() { - t.TaskResult.ErrResult() - return t.TaskResult - } - - t.TaskResult.NormalResult() - return t.TaskResult -} - -func (t *RemoteTask) RunWithTimeout(ctx context.Context, runtime connector.Runtime, host connector.Host, index int, - wg *sync.WaitGroup, pool chan struct{}) { - - pool <- struct{}{} - - resCh := make(chan error) - go t.Run(runtime, host, index, resCh) - - select { - case <-ctx.Done(): - t.TaskResult.AppendErr(host, fmt.Errorf("execute task timeout, Timeout=%s", util.ShortDur(t.Timeout))) - case e := <-resCh: - if e != nil { - t.TaskResult.AppendErr(host, e) - } - } - - <-pool - wg.Done() -} - -func (t *RemoteTask) Run(runtime connector.Runtime, host connector.Host, index int, resCh chan error) { - var res error - defer func() { - //runtime.GetConnector().Close(host) - - resCh <- res - close(resCh) - }() - - if err := t.ConfigureSelfRuntime(runtime, host, index); err != nil { - res = err - return - } - - t.Prepare.Init(t.ModuleCache, t.PipelineCache) - t.Prepare.AutoAssert(runtime) - if ok, err := t.WhenWithRetry(runtime); !ok { - if err != nil { - res = err - return - } else { - t.TaskResult.AppendSkip(host) - return - } - } - - t.Action.Init(t.ModuleCache, t.PipelineCache) - t.Action.AutoAssert(runtime) - if err := t.ExecuteWithRetry(runtime); err != nil { - res = err - return - } - - t.TaskResult.AppendSuccess(host) - return -} - -func (t *RemoteTask) ConfigureSelfRuntime(runtime connector.Runtime, host connector.Host, index int) error { - conn, err := runtime.GetConnector().Connect(host) - if err != nil { - return errors.Wrapf(err, "failed to connect to %s", host.GetAddress()) - } - - r := &connector.Runner{ - Conn: conn, - //Debug: runtime.Arg.Debug, - Host: host, - Index: index, - } - runtime.SetRunner(r) - return nil -} - -func (t *RemoteTask) When(runtime connector.Runtime) (bool, error) { - if t.Prepare == nil { - return true, nil - } - if ok, err := t.Prepare.PreCheck(runtime); err != nil { - return false, err - } else if !ok { - return false, nil - } - return true, nil -} - -func (t *RemoteTask) WhenWithRetry(runtime connector.Runtime) (bool, error) { - pass := false - err := fmt.Errorf("pre-check exec failed after %d retries", t.Retry) - for i := 0; i < t.Retry; i++ { - if res, e := t.When(runtime); e != nil { - logger.Log.Messagef(runtime.RemoteHost().GetName(), e.Error()) - - if i == t.Retry-1 { - err = errors.New(err.Error() + e.Error()) - continue - } - logger.Log.Infof("retry: [%s]", runtime.GetRunner().Host.GetName()) - time.Sleep(t.Delay) - continue - } else { - err = nil - pass = res - break - } - } - - return pass, err -} - -func (t *RemoteTask) ExecuteWithRetry(runtime connector.Runtime) error { - err := fmt.Errorf("[%s] exec failed after %d retries: ", t.Name, t.Retry) - for i := 0; i < t.Retry; i++ { - e := t.Action.Execute(runtime) - if e != nil { - logger.Log.Messagef(runtime.RemoteHost().GetName(), e.Error()) - - if i == t.Retry-1 { - err = errors.New(err.Error() + e.Error()) - continue - } - logger.Log.Infof("retry: [%s]", runtime.GetRunner().Host.GetName()) - time.Sleep(t.Delay) - continue - } else { - err = nil - break - } - } - return err -} - -func (t *RemoteTask) ExecuteRollback() { - if t.Rollback == nil { - return - } - if !t.TaskResult.IsFailed() { - return - } - - ctx, cancel := context.WithTimeout(context.Background(), t.Timeout) - defer cancel() - routinePool := make(chan struct{}, DefaultCon) - defer close(routinePool) - - rwg := &sync.WaitGroup{} - for i, ar := range t.TaskResult.ActionResults { - if ar.Host == nil || t.Runtime.HostIsDeprecated(ar.Host) { - continue - } - selfRuntime := t.Runtime.Copy() - - if t.Parallel { - go t.RollbackWithTimeout(ctx, selfRuntime, ar.Host, i, ar, rwg, routinePool) - } else { - t.RollbackWithTimeout(ctx, selfRuntime, ar.Host, i, ar, rwg, routinePool) - } - rwg.Add(1) - } - rwg.Wait() -} - -func (t *RemoteTask) RollbackWithTimeout(ctx context.Context, runtime connector.Runtime, host connector.Host, index int, - result *ending.ActionResult, wg *sync.WaitGroup, pool chan struct{}) { - - pool <- struct{}{} - - resCh := make(chan error) - go t.RunRollback(runtime, host, index, result, resCh) - - select { - case <-ctx.Done(): - logger.Log.Errorf("rollback-failed: [%s]", runtime.GetRunner().Host.GetName()) - logger.Log.Messagef(runtime.RemoteHost().GetName(), fmt.Sprintf("execute task timeout, Timeout=%d", t.Timeout)) - case e := <-resCh: - if e != nil { - logger.Log.Errorf("rollback-failed: [%s]", runtime.GetRunner().Host.GetName()) - logger.Log.Messagef(runtime.RemoteHost().GetName(), e.Error()) - } - } - - <-pool - wg.Done() -} - -func (t *RemoteTask) RunRollback(runtime connector.Runtime, host connector.Host, index int, result *ending.ActionResult, resCh chan error) { - var res error - defer func() { - //runtime.GetConnector().Close(host) - - resCh <- res - close(resCh) - }() - - if err := t.ConfigureSelfRuntime(runtime, host, index); err != nil { - res = err - return - } - - logger.Log.Infof("rollback: [%s]", runtime.GetRunner().Host.GetName()) - - t.Rollback.Init(t.ModuleCache, t.PipelineCache) - t.Rollback.AutoAssert(runtime) - if err := t.Rollback.Execute(runtime, result); err != nil { - res = err - return - } - return -} - -func (t *RemoteTask) Default() { - t.TaskResult = ending.NewTaskResult() - if t.Name == "" { - t.Name = DefaultTaskName - } - - if t.Prepare == nil { - t.Prepare = new(prepare.BasePrepare) - } - - if t.Retry <= 0 { - t.Retry = 3 - } - - if t.Delay <= 0 { - t.Delay = 5 * time.Second - } - - if t.Timeout <= 0 { - t.Timeout = DefaultTimeout * time.Minute - } - - if t.Concurrency <= 0 || t.Concurrency > 1 { - t.Concurrency = 1 - } -} - -func (t *RemoteTask) calculateConcurrency() int { - num := t.Concurrency * float64(len(t.Hosts)) - res := int(util.Round(num, 0)) - if res < 1 { - res = 1 - } - return res -} diff --git a/cmd/kk/pkg/core/task/remote_task_test.go b/cmd/kk/pkg/core/task/remote_task_test.go deleted file mode 100644 index ad80f9d20..000000000 --- a/cmd/kk/pkg/core/task/remote_task_test.go +++ /dev/null @@ -1,141 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package task - -import ( - "testing" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -func TestTask_calculateConcurrency(t1 *testing.T) { - type fields struct { - Hosts []connector.BaseHost - Concurrency float64 - } - tests := []struct { - name string - fields fields - want int - }{ - { - name: "test1", - fields: fields{ - Concurrency: 0.5, - Hosts: []connector.BaseHost{ - {Name: "node1"}, - {Name: "node2"}, - {Name: "node3"}, - }, - }, - want: 2, - }, - { - name: "test2", - fields: fields{ - Concurrency: 0.5, - Hosts: []connector.BaseHost{ - {Name: "node1"}, - {Name: "node2"}, - {Name: "node3"}, - {Name: "node4"}, - }, - }, - want: 2, - }, - { - name: "test3", - fields: fields{ - Concurrency: 0.4, - Hosts: []connector.BaseHost{ - {Name: "node1"}, - {Name: "node2"}, - {Name: "node3"}, - }, - }, - want: 1, - }, - { - name: "test4", - fields: fields{ - Concurrency: 0.4, - Hosts: []connector.BaseHost{ - {Name: "node1"}, - {Name: "node2"}, - {Name: "node3"}, - {Name: "node4"}, - }, - }, - want: 2, - }, - { - name: "test5", - fields: fields{ - Concurrency: 0.1, - Hosts: []connector.BaseHost{ - {Name: "node1"}, - {Name: "node2"}, - {Name: "node3"}, - {Name: "node4"}, - }, - }, - want: 1, - }, - { - name: "test6", - fields: fields{ - Concurrency: 0.222222222222222, - Hosts: []connector.BaseHost{ - {Name: "node1"}, - {Name: "node2"}, - {Name: "node3"}, - {Name: "node4"}, - }, - }, - want: 1, - }, - { - name: "test7", - fields: fields{ - Concurrency: 1, - Hosts: []connector.BaseHost{ - {Name: "node1"}, - {Name: "node2"}, - {Name: "node3"}, - {Name: "node4"}, - }, - }, - want: 4, - }, - } - for _, tt := range tests { - t1.Run(tt.name, func(t1 *testing.T) { - var hosts []connector.Host - for _, v := range tt.fields.Hosts { - hosts = append(hosts, &v) - } - - t := &RemoteTask{ - Concurrency: tt.fields.Concurrency, - Hosts: hosts, - } - if got := t.calculateConcurrency(); got != tt.want { - t1.Errorf("calculateConcurrency() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/cmd/kk/pkg/core/util/file.go b/cmd/kk/pkg/core/util/file.go deleted file mode 100644 index cdb3e36cf..000000000 --- a/cmd/kk/pkg/core/util/file.go +++ /dev/null @@ -1,243 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package util - -import ( - "archive/tar" - "compress/gzip" - "crypto/md5" - "fmt" - "io" - "io/fs" - "os" - "path/filepath" - "strings" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" -) - -func IsExist(path string) bool { - _, err := os.Stat(path) - if err != nil { - if os.IsExist(err) { - return true - } - if os.IsNotExist(err) { - return false - } - return false - } - return true -} - -func CreateDir(path string) error { - if IsExist(path) == false { - err := os.MkdirAll(path, os.ModePerm) - if err != nil { - return err - } - } - return nil -} - -func IsDir(path string) bool { - s, err := os.Stat(path) - if err != nil { - return false - } - return s.IsDir() -} - -func CountDirFiles(dirName string) int { - if !IsDir(dirName) { - return 0 - } - var count int - err := filepath.Walk(dirName, func(path string, info os.FileInfo, err error) error { - if info.IsDir() { - return nil - } - count++ - return nil - }) - if err != nil { - logger.Log.Fatalln("count dir files failed %v", err) - return 0 - } - return count -} - -// FileMD5 count file md5 -func FileMD5(path string) (string, error) { - file, err := os.Open(path) - if err != nil { - return "", err - } - - m := md5.New() - if _, err := io.Copy(m, file); err != nil { - return "", err - } - - fileMd5 := fmt.Sprintf("%x", m.Sum(nil)) - return fileMd5, nil -} - -func LocalMd5Sum(src string) string { - md5Str, err := FileMD5(src) - if err != nil { - logger.Log.Fatalf("get file md5 failed %v", err) - return "" - } - return md5Str -} - -// MkFileFullPathDir is used to file create the full path. -// eg. there is a file "./aa/bb/xxx.txt", and dir ./aa/bb is not exist, and will create the full path dir. -func MkFileFullPathDir(fileName string) error { - localDir := filepath.Dir(fileName) - err := Mkdir(localDir) - if err != nil { - return fmt.Errorf("create local dir %s failed: %v", localDir, err) - } - return nil -} - -func Mkdir(dirName string) error { - return os.MkdirAll(dirName, os.ModePerm) -} - -func WriteFile(fileName string, content []byte) error { - dir := filepath.Dir(fileName) - if _, err := os.Stat(dir); os.IsNotExist(err) { - if err = os.MkdirAll(dir, common.FileMode0755); err != nil { - return err - } - } - - if err := os.WriteFile(fileName, content, common.FileMode0644); err != nil { - return err - } - return nil -} - -func Tar(src, dst, trimPrefix string) error { - fw, err := os.Create(dst) - if err != nil { - return err - } - defer fw.Close() - - gw := gzip.NewWriter(fw) - defer gw.Close() - - tw := tar.NewWriter(gw) - defer tw.Close() - - return filepath.Walk(src, func(path string, info fs.FileInfo, err error) error { - if err != nil { - return err - } - - hdr, err := tar.FileInfoHeader(info, "") - if err != nil { - return err - } - - if !info.Mode().IsRegular() { - return nil - } - - fr, err := os.Open(path) - defer fr.Close() - if err != nil { - return err - } - - path = strings.TrimPrefix(path, trimPrefix) - fmt.Println(strings.TrimPrefix(path, string(filepath.Separator))) - - hdr.Name = strings.TrimPrefix(path, string(filepath.Separator)) - if err := tw.WriteHeader(hdr); err != nil { - return err - } - - if _, err := io.Copy(tw, fr); err != nil { - return err - } - - return nil - }) -} - -func Untar(src, dst string) error { - fr, err := os.Open(src) - if err != nil { - return err - } - defer fr.Close() - - gr, err := gzip.NewReader(fr) - if err != nil { - return err - } - defer gr.Close() - - tr := tar.NewReader(gr) - for { - hdr, err := tr.Next() - - switch { - case err == io.EOF: - return nil - case err != nil: - return err - case hdr == nil: - continue - } - - dstPath := filepath.Join(dst, hdr.Name) - - switch hdr.Typeflag { - case tar.TypeDir: - if !IsExist(dstPath) && IsDir(dstPath) { - if err := CreateDir(dstPath); err != nil { - return err - } - } - case tar.TypeReg: - if dir := filepath.Dir(dstPath); !IsExist(dir) { - if err := CreateDir(dir); err != nil { - return err - } - } - - file, err := os.OpenFile(dstPath, os.O_CREATE|os.O_RDWR, os.FileMode(hdr.Mode)) - if err != nil { - return err - } - - if _, err = io.Copy(file, tr); err != nil { - return err - } - - fmt.Println(dstPath) - file.Close() - } - } -} diff --git a/cmd/kk/pkg/core/util/ip.go b/cmd/kk/pkg/core/util/ip.go deleted file mode 100644 index 162650a80..000000000 --- a/cmd/kk/pkg/core/util/ip.go +++ /dev/null @@ -1,168 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package util - -import ( - "encoding/binary" - "net" - "strconv" - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" -) - -func ParseIp(ip string) []string { - var availableIPs []string - // if ip is "1.1.1.1/",trim / - ip = strings.TrimRight(ip, "/") - if strings.Contains(ip, "/") == true { - if strings.Contains(ip, "/32") == true { - aip := strings.Replace(ip, "/32", "", -1) - availableIPs = append(availableIPs, aip) - } else { - availableIPs = GetAvailableIP(ip) - } - } else if strings.Contains(ip, "-") == true { - ipRange := strings.SplitN(ip, "-", 2) - availableIPs = GetAvailableIPRange(ipRange[0], ipRange[1]) - } else { - availableIPs = append(availableIPs, ip) - } - return availableIPs -} - -func GetAvailableIPRange(ipStart, ipEnd string) []string { - var availableIPs []string - - firstIP := net.ParseIP(ipStart) - endIP := net.ParseIP(ipEnd) - if firstIP.To4() == nil || endIP.To4() == nil { - return availableIPs - } - firstIPNum := ipToInt(firstIP.To4()) - EndIPNum := ipToInt(endIP.To4()) - pos := int32(1) - - newNum := firstIPNum - - for newNum <= EndIPNum { - availableIPs = append(availableIPs, intToIP(newNum).String()) - newNum = newNum + pos - } - return availableIPs -} - -func GetAvailableIP(ipAndMask string) []string { - var availableIPs []string - - ipAndMask = strings.TrimSpace(ipAndMask) - ipAndMask = IPAddressToCIDR(ipAndMask) - _, ipnet, _ := net.ParseCIDR(ipAndMask) - - firstIP, _ := networkRange(ipnet) - ipNum := ipToInt(firstIP) - size := networkSize(ipnet.Mask) - pos := int32(1) - max := size - 2 // -1 for the broadcast address, -1 for the gateway address - - var newNum int32 - for attempt := int32(0); attempt < max; attempt++ { - newNum = ipNum + pos - pos = pos%max + 1 - availableIPs = append(availableIPs, intToIP(newNum).String()) - } - return availableIPs -} - -func IPAddressToCIDR(ipAddress string) string { - if strings.Contains(ipAddress, "/") == true { - ipAndMask := strings.Split(ipAddress, "/") - ip := ipAndMask[0] - mask := ipAndMask[1] - if strings.Contains(mask, ".") == true { - mask = IPMaskStringToCIDR(mask) - } - return ip + "/" + mask - } else { - return ipAddress - } -} - -func IPMaskStringToCIDR(netmask string) string { - netmaskList := strings.Split(netmask, ".") - var mint []int - for _, v := range netmaskList { - strv, _ := strconv.Atoi(v) - mint = append(mint, strv) - } - myIPMask := net.IPv4Mask(byte(mint[0]), byte(mint[1]), byte(mint[2]), byte(mint[3])) - ones, _ := myIPMask.Size() - return strconv.Itoa(ones) -} - -func networkRange(network *net.IPNet) (net.IP, net.IP) { - netIP := network.IP.To4() - firstIP := netIP.Mask(network.Mask) - lastIP := net.IPv4(0, 0, 0, 0).To4() - for i := 0; i < len(lastIP); i++ { - lastIP[i] = netIP[i] | ^network.Mask[i] - } - return firstIP, lastIP -} - -func networkSize(mask net.IPMask) int32 { - m := net.IPv4Mask(0, 0, 0, 0) - for i := 0; i < net.IPv4len; i++ { - m[i] = ^mask[i] - } - return int32(binary.BigEndian.Uint32(m)) + 1 -} - -func ipToInt(ip net.IP) int32 { - return int32(binary.BigEndian.Uint32(ip.To4())) -} - -func intToIP(n int32) net.IP { - b := make([]byte, 4) - binary.BigEndian.PutUint32(b, uint32(n)) - return net.IP(b) -} - -func GetLocalIP() (string, error) { - addrs, err := net.InterfaceAddrs() - if err != nil { - return "", err - } - for _, addr := range addrs { - if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { - if ipnet.IP.To4() != nil && ipnet.IP.IsGlobalUnicast() { - return ipnet.IP.String(), nil - } - } - } - return "", errors.New("valid local IP not found!") -} - -func LocalIP() string { - localIp, err := GetLocalIP() - if err != nil { - logger.Log.Fatalf("Failed to get Local IP: %v", err) - } - return localIp -} diff --git a/cmd/kk/pkg/core/util/time.go b/cmd/kk/pkg/core/util/time.go deleted file mode 100644 index 5f1d137f1..000000000 --- a/cmd/kk/pkg/core/util/time.go +++ /dev/null @@ -1,33 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package util - -import ( - "strings" - "time" -) - -func ShortDur(d time.Duration) string { - s := d.String() - if strings.HasSuffix(s, "m0s") { - s = s[:len(s)-2] - } - if strings.HasSuffix(s, "h0m") { - s = s[:len(s)-2] - } - return s -} diff --git a/cmd/kk/pkg/core/util/time_test.go b/cmd/kk/pkg/core/util/time_test.go deleted file mode 100644 index 7a18073b6..000000000 --- a/cmd/kk/pkg/core/util/time_test.go +++ /dev/null @@ -1,61 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package util - -import ( - "testing" - "time" -) - -func TestShortDur(t *testing.T) { - tests := []struct { - d time.Duration - want string - }{ - { - d: 3 * time.Second, - want: "3s", - }, - { - d: 60 * time.Second, - want: "1m", - }, - { - d: 4 * time.Minute, - want: "4m", - }, - { - d: 1*time.Hour + 3*time.Second, - want: "1h0m3s", - }, - { - d: 1*time.Hour + 4*time.Minute, - want: "1h4m", - }, - { - d: 1*time.Hour + 4*time.Minute + 3*time.Second, - want: "1h4m3s", - }, - } - for _, tt := range tests { - t.Run("", func(t *testing.T) { - if got := ShortDur(tt.d); got != tt.want { - t.Errorf("ShortDur() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/cmd/kk/pkg/core/util/util.go b/cmd/kk/pkg/core/util/util.go deleted file mode 100644 index d14c0f6fd..000000000 --- a/cmd/kk/pkg/core/util/util.go +++ /dev/null @@ -1,138 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package util - -import ( - "bytes" - "fmt" - "math" - "os" - "os/exec" - "os/user" - "runtime" - "sort" - "strings" - "text/template" - - "github.com/pkg/errors" -) - -type Data map[string]interface{} - -// Render text template with given `variables` Render-context -func Render(tmpl *template.Template, variables map[string]interface{}) (string, error) { - - var buf strings.Builder - - if err := tmpl.Execute(&buf, variables); err != nil { - return "", errors.Wrap(err, "Failed to render template") - } - return buf.String(), nil -} - -// Home returns the home directory for the executing user. -func Home() (string, error) { - u, err := user.Current() - if nil == err { - return u.HomeDir, nil - } - - if "windows" == runtime.GOOS { - return homeWindows() - } - - return homeUnix() -} - -func homeUnix() (string, error) { - if home := os.Getenv("HOME"); home != "" { - return home, nil - } - - var stdout bytes.Buffer - cmd := exec.Command("sh", "-c", "eval echo ~$USER") - cmd.Stdout = &stdout - if err := cmd.Run(); err != nil { - return "", err - } - - result := strings.TrimSpace(stdout.String()) - if result == "" { - return "", errors.New("blank output when reading home directory") - } - - return result, nil -} - -func homeWindows() (string, error) { - drive := os.Getenv("HOMEDRIVE") - path := os.Getenv("HOMEPATH") - home := drive + path - if drive == "" || path == "" { - home = os.Getenv("USERPROFILE") - } - if home == "" { - return "", errors.New("HOMEDRIVE, HOMEPATH, and USERPROFILE are blank") - } - - return home, nil -} - -func GetArgs(argsMap map[string]string, args []string) ([]string, map[string]string) { - targetMap := make(map[string]string, len(argsMap)) - for k, v := range argsMap { - targetMap[k] = v - } - targetSlice := make([]string, len(args)) - copy(targetSlice, args) - - for _, arg := range targetSlice { - splitArg := strings.SplitN(arg, "=", 2) - if len(splitArg) < 2 { - continue - } - targetMap[splitArg[0]] = splitArg[1] - } - - for arg, value := range targetMap { - cmd := fmt.Sprintf("%s=%s", arg, value) - targetSlice = append(targetSlice, cmd) - } - sort.Strings(targetSlice) - return targetSlice, targetMap -} - -// Round returns the result of rounding 'val' according to the specified 'precision' precision (the number of digits after the decimal point)。 -// and precision can be negative number or zero -func Round(val float64, precision int) float64 { - p := math.Pow10(precision) - return math.Floor(val*p+0.5) / p -} - -// ArchAlias returns the alias of cpu's architecture. -// amd64: x86_64 -// arm64: aarch64 -func ArchAlias(arch string) string { - switch arch { - case "amd64": - return "x86_64" - case "arm64": - return "aarch64" - default: - return "" - } -} diff --git a/cmd/kk/pkg/core/util/util_test.go b/cmd/kk/pkg/core/util/util_test.go deleted file mode 100644 index 8351d73e0..000000000 --- a/cmd/kk/pkg/core/util/util_test.go +++ /dev/null @@ -1,55 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package util - -import "testing" - -func TestRound(t *testing.T) { - type args struct { - val float64 - precision int - } - tests := []struct { - name string - args args - want float64 - }{ - { - name: "test1", - args: args{ - val: 0.5555, - precision: 0, - }, - want: 1, - }, - { - name: "test2", - args: args{ - val: 0.4555, - precision: 0, - }, - want: 0, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := Round(tt.args.val, tt.args.precision); got != tt.want { - t.Errorf("Round() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/cmd/kk/pkg/etcd/certs.go b/cmd/kk/pkg/etcd/certs.go deleted file mode 100644 index d481ae1d0..000000000 --- a/cmd/kk/pkg/etcd/certs.go +++ /dev/null @@ -1,262 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package etcd - -import ( - "crypto/x509" - "fmt" - "net" - "os" - "path/filepath" - "strings" - - "github.com/pkg/errors" - "k8s.io/client-go/util/cert" - certutil "k8s.io/client-go/util/cert" - netutils "k8s.io/utils/net" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils/certs" -) - -// KubekeyCertEtcdCA is the definition of the root CA used by the hosted etcd server. -func KubekeyCertEtcdCA() *certs.KubekeyCert { - return &certs.KubekeyCert{ - Name: "etcd-ca", - LongName: "self-signed CA to provision identities for etcd", - BaseName: "ca", - Config: certs.CertConfig{ - Config: certutil.Config{ - CommonName: "etcd-ca", - }, - }, - } -} - -// KubekeyCertEtcdAdmin is the definition of the cert for etcd admin. -func KubekeyCertEtcdAdmin(hostname string, altNames *certutil.AltNames) *certs.KubekeyCert { - l := strings.Split(hostname, ".") - return &certs.KubekeyCert{ - Name: "etcd-admin", - LongName: "certificate for etcd admin", - BaseName: fmt.Sprintf("admin-%s", hostname), - CAName: "etcd-ca", - Config: certs.CertConfig{ - Config: certutil.Config{ - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, - AltNames: *altNames, - CommonName: fmt.Sprintf("etcd-admin-%s", l[0]), - }, - }, - } -} - -// KubekeyCertEtcdMember is the definition of the cert for etcd member. -func KubekeyCertEtcdMember(hostname string, altNames *certutil.AltNames) *certs.KubekeyCert { - l := strings.Split(hostname, ".") - return &certs.KubekeyCert{ - Name: "etcd-member", - LongName: "certificate for etcd member", - BaseName: fmt.Sprintf("member-%s", hostname), - CAName: "etcd-ca", - Config: certs.CertConfig{ - Config: certutil.Config{ - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, - AltNames: *altNames, - CommonName: fmt.Sprintf("etcd-member-%s", l[0]), - }, - }, - } -} - -// KubekeyCertEtcdClient is the definition of the cert for etcd client. -func KubekeyCertEtcdClient(hostname string, altNames *certutil.AltNames) *certs.KubekeyCert { - l := strings.Split(hostname, ".") - return &certs.KubekeyCert{ - Name: "etcd-client", - LongName: "certificate for etcd client", - BaseName: fmt.Sprintf("node-%s", hostname), - CAName: "etcd-ca", - Config: certs.CertConfig{ - Config: certutil.Config{ - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, - AltNames: *altNames, - CommonName: fmt.Sprintf("etcd-node-%s", l[0]), - }, - }, - } -} - -type FetchCerts struct { - common.KubeAction -} - -func (f *FetchCerts) Execute(runtime connector.Runtime) error { - src := "/etc/ssl/etcd/ssl" - dst := fmt.Sprintf("%s/pki/etcd", runtime.GetWorkDir()) - - v, ok := f.PipelineCache.Get(common.ETCDCluster) - if !ok { - return errors.New("get etcd status from pipeline cache failed") - } - - c := v.(*EtcdCluster) - - if c.clusterExist { - certs, err := runtime.GetRunner().SudoCmd("ls /etc/ssl/etcd/ssl/ | grep .pem", false) - if err != nil { - return errors.Wrap(err, "failed to find certificate files") - } - - certsList := strings.Split(certs, "\r\n") - if len(certsList) > 0 { - for _, cert := range certsList { - if err := runtime.GetRunner().Fetch(filepath.Join(dst, cert), filepath.Join(src, cert)); err != nil { - return errors.Wrap(err, fmt.Sprintf("Fetch %s failed", filepath.Join(src, cert))) - } - } - } - } - - return nil -} - -type GenerateCerts struct { - common.KubeAction -} - -func (g *GenerateCerts) Execute(runtime connector.Runtime) error { - - pkiPath := fmt.Sprintf("%s/pki/etcd", runtime.GetWorkDir()) - - altName := GenerateAltName(g.KubeConf, &runtime) - - files := []string{"ca.pem", "ca-key.pem"} - - // CA - certsList := []*certs.KubekeyCert{KubekeyCertEtcdCA()} - - // Certs - for _, host := range runtime.GetAllHosts() { - if host.IsRole(common.ETCD) { - certsList = append(certsList, KubekeyCertEtcdAdmin(host.GetName(), altName)) - files = append(files, []string{fmt.Sprintf("admin-%s.pem", host.GetName()), fmt.Sprintf("admin-%s-key.pem", host.GetName())}...) - certsList = append(certsList, KubekeyCertEtcdMember(host.GetName(), altName)) - files = append(files, []string{fmt.Sprintf("member-%s.pem", host.GetName()), fmt.Sprintf("member-%s-key.pem", host.GetName())}...) - } - if host.IsRole(common.Master) { - certsList = append(certsList, KubekeyCertEtcdClient(host.GetName(), altName)) - files = append(files, []string{fmt.Sprintf("node-%s.pem", host.GetName()), fmt.Sprintf("node-%s-key.pem", host.GetName())}...) - } - } - - var lastCACert *certs.KubekeyCert - for _, c := range certsList { - if c.CAName == "" { - err := certs.GenerateCA(c, pkiPath, g.KubeConf) - if err != nil { - return err - } - lastCACert = c - } else { - err := certs.GenerateCerts(c, lastCACert, pkiPath, g.KubeConf) - if err != nil { - return err - } - } - } - - g.ModuleCache.Set(LocalCertsDir, pkiPath) - g.ModuleCache.Set(CertsFileList, files) - - return nil -} - -func GenerateAltName(k *common.KubeConf, runtime *connector.Runtime) *cert.AltNames { - var altName cert.AltNames - - dnsList := []string{"localhost", "etcd.kube-system.svc.cluster.local", "etcd.kube-system.svc", "etcd.kube-system", "etcd"} - ipList := []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback} - - if k.Cluster.ControlPlaneEndpoint.Domain == "" { - dnsList = append(dnsList, kubekeyapiv1alpha2.DefaultLBDomain) - } else { - dnsList = append(dnsList, k.Cluster.ControlPlaneEndpoint.Domain) - } - - for _, host := range k.Cluster.Hosts { - dnsList = append(dnsList, host.Name) - internalAddress := netutils.ParseIPSloppy(host.InternalAddress) - if internalAddress != nil { - ipList = append(ipList, internalAddress) - } - } - - altName.DNSNames = dnsList - altName.IPs = ipList - - return &altName -} - -type FetchCertsForExternalEtcd struct { - common.KubeAction -} - -func (f *FetchCertsForExternalEtcd) Execute(runtime connector.Runtime) error { - - pkiPath := fmt.Sprintf("%s/pki/etcd", runtime.GetWorkDir()) - - if err := util.CreateDir(pkiPath); err != nil { - return errors.Wrap(err, fmt.Sprintf("failed to create dir %s", pkiPath)) - } - - srcCertsFiles := []string{f.KubeConf.Cluster.Etcd.External.CAFile, f.KubeConf.Cluster.Etcd.External.CertFile, f.KubeConf.Cluster.Etcd.External.KeyFile} - dstCertsFiles := []string{} - for _, certFile := range srcCertsFiles { - if len(certFile) != 0 { - certPath, err := filepath.Abs(certFile) - if err != nil { - return errors.Wrap(err, "bad certificate file path") - } - _, err = os.Stat(certPath) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("%s does not exist", certPath)) - } - - dstCertFileName := filepath.Base(certPath) - dstCert := fmt.Sprintf("%s/%s", pkiPath, dstCertFileName) - dstCertsFiles = append(dstCertsFiles, dstCertFileName) - - data, err := os.ReadFile(certPath) - if err != nil { - return errors.Wrap(err, "failed to copy certificate content") - } - - if err := os.WriteFile(dstCert, data, 0600); err != nil { - return errors.Wrap(err, "failed to copy certificate content") - } - } - } - - f.ModuleCache.Set(LocalCertsDir, pkiPath) - f.ModuleCache.Set(CertsFileList, dstCertsFiles) - - return nil -} diff --git a/cmd/kk/pkg/etcd/module.go b/cmd/kk/pkg/etcd/module.go deleted file mode 100644 index f454cba03..000000000 --- a/cmd/kk/pkg/etcd/module.go +++ /dev/null @@ -1,425 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package etcd - -import ( - "path/filepath" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/etcd/templates" -) - -type PreCheckModule struct { - common.KubeModule - Skip bool -} - -func (p *PreCheckModule) IsSkip() bool { - return p.Skip -} - -func (p *PreCheckModule) Init() { - p.Name = "ETCDPreCheckModule" - p.Desc = "Get ETCD cluster status" - - getStatus := &task.RemoteTask{ - Name: "GetETCDStatus", - Desc: "Get etcd status", - Hosts: p.Runtime.GetHostsByRole(common.ETCD), - Action: new(GetStatus), - Parallel: false, - Retry: 0, - } - p.Tasks = []task.Interface{ - getStatus, - } -} - -type CertsModule struct { - common.KubeModule - Skip bool -} - -func (p *CertsModule) IsSkip() bool { - return p.Skip -} - -func (c *CertsModule) Init() { - c.Name = "CertsModule" - c.Desc = "Sign ETCD cluster certs" - - switch c.KubeConf.Cluster.Etcd.Type { - case kubekeyapiv1alpha2.KubeKey: - c.Tasks = CertsModuleForKubeKey(c) - case kubekeyapiv1alpha2.External: - c.Tasks = CertsModuleForExternal(c) - } -} - -func CertsModuleForKubeKey(c *CertsModule) []task.Interface { - // If the etcd cluster already exists, obtain the certificate in use from the etcd node. - fetchCerts := &task.RemoteTask{ - Name: "FetchETCDCerts", - Desc: "Fetch etcd certs", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Prepare: new(FirstETCDNode), - Action: new(FetchCerts), - Parallel: false, - } - - generateCerts := &task.LocalTask{ - Name: "GenerateETCDCerts", - Desc: "Generate etcd Certs", - Action: new(GenerateCerts), - } - - syncCertsFile := &task.RemoteTask{ - Name: "SyncCertsFile", - Desc: "Synchronize certs file", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Action: new(SyncCertsFile), - Parallel: true, - Retry: 1, - } - - syncCertsToMaster := &task.RemoteTask{ - Name: "SyncCertsFileToMaster", - Desc: "Synchronize certs file to master", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &common.OnlyETCD{Not: true}, - Action: new(SyncCertsFile), - Parallel: true, - Retry: 1, - } - - return []task.Interface{ - fetchCerts, - generateCerts, - syncCertsFile, - syncCertsToMaster, - } -} - -func CertsModuleForExternal(c *CertsModule) []task.Interface { - fetchCerts := &task.LocalTask{ - Name: "FetchETCDCerts", - Desc: "Fetch etcd certs", - Action: new(FetchCertsForExternalEtcd), - } - - syncCertsToMaster := &task.RemoteTask{ - Name: "SyncCertsFileToMaster", - Desc: "Synchronize certs file to master", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Action: new(SyncCertsFile), - Parallel: true, - Retry: 1, - } - - return []task.Interface{ - fetchCerts, - syncCertsToMaster, - } -} - -type InstallETCDBinaryModule struct { - common.KubeModule - Skip bool -} - -func (p *InstallETCDBinaryModule) IsSkip() bool { - return p.Skip -} - -func (i *InstallETCDBinaryModule) Init() { - i.Name = "InstallETCDBinaryModule" - i.Desc = "Install ETCD cluster" - - installETCDBinary := &task.RemoteTask{ - Name: "InstallETCDBinary", - Desc: "Install etcd using binary", - Hosts: i.Runtime.GetHostsByRole(common.ETCD), - Action: new(InstallETCDBinary), - Parallel: true, - Retry: 1, - } - - generateETCDService := &task.RemoteTask{ - Name: "GenerateETCDService", - Desc: "Generate etcd service", - Hosts: i.Runtime.GetHostsByRole(common.ETCD), - Action: &action.Template{ - Template: templates.ETCDService, - Dst: "/etc/systemd/system/etcd.service", - }, - Parallel: true, - Retry: 1, - } - - accessAddress := &task.RemoteTask{ - Name: "GenerateAccessAddress", - Desc: "Generate access address", - Hosts: i.Runtime.GetHostsByRole(common.ETCD), - Prepare: new(FirstETCDNode), - Action: new(GenerateAccessAddress), - Parallel: true, - Retry: 1, - } - - i.Tasks = []task.Interface{ - installETCDBinary, - generateETCDService, - accessAddress, - } -} - -type ConfigureModule struct { - common.KubeModule - Skip bool -} - -func (p *ConfigureModule) IsSkip() bool { - return p.Skip -} - -func (e *ConfigureModule) Init() { - e.Name = "ETCDConfigureModule" - e.Desc = "Configure ETCD cluster" - - if v, ok := e.PipelineCache.Get(common.ETCDCluster); ok { - cluster := v.(*EtcdCluster) - if !cluster.clusterExist { - e.Tasks = handleNewCluster(e) - } else { - e.Tasks = handleExistCluster(e) - } - } -} - -func handleNewCluster(c *ConfigureModule) []task.Interface { - - existETCDHealthCheck := &task.RemoteTask{ - Name: "ExistETCDHealthCheck", - Desc: "Health check on exist etcd", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Prepare: new(NodeETCDExist), - Action: new(HealthCheck), - Parallel: true, - Retry: 20, - } - - generateETCDConfig := &task.RemoteTask{ - Name: "GenerateETCDConfig", - Desc: "Generate etcd.env config on new etcd", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Prepare: &NodeETCDExist{Not: true}, - Action: new(GenerateConfig), - Parallel: false, - } - - allRefreshETCDConfig := &task.RemoteTask{ - Name: "AllRefreshETCDConfig", - Desc: "Refresh etcd.env config on all etcd", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Action: new(RefreshConfig), - Parallel: false, - } - - restart := &task.RemoteTask{ - Name: "RestartETCD", - Desc: "Restart etcd", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Prepare: &NodeETCDExist{Not: true}, - Action: new(RestartETCD), - Parallel: true, - } - - allETCDNodeHealthCheck := &task.RemoteTask{ - Name: "AllETCDNodeHealthCheck", - Desc: "Health check on all etcd", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Action: new(HealthCheck), - Parallel: true, - Retry: 20, - } - - refreshETCDConfigToExist := &task.RemoteTask{ - Name: "RefreshETCDConfigToExist", - Desc: "Refresh etcd.env config to exist mode on all etcd", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Action: &RefreshConfig{ToExisting: true}, - Parallel: false, - } - - tasks := []task.Interface{ - existETCDHealthCheck, - generateETCDConfig, - allRefreshETCDConfig, - restart, - allETCDNodeHealthCheck, - refreshETCDConfigToExist, - allETCDNodeHealthCheck, - } - return tasks -} - -func handleExistCluster(c *ConfigureModule) []task.Interface { - - existETCDHealthCheck := &task.RemoteTask{ - Name: "ExistETCDHealthCheck", - Desc: "Health check on exist etcd", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Prepare: new(NodeETCDExist), - Action: new(HealthCheck), - Parallel: true, - Retry: 20, - } - - generateETCDConfig := &task.RemoteTask{ - Name: "GenerateETCDConfig", - Desc: "Generate etcd.env config on new etcd", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Prepare: &NodeETCDExist{Not: true}, - Action: new(GenerateConfig), - Parallel: false, - } - - joinMember := &task.RemoteTask{ - Name: "JoinETCDMember", - Desc: "Join etcd member", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Prepare: &NodeETCDExist{Not: true}, - Action: new(JoinMember), - Parallel: false, - } - - newETCDNodeHealthCheck := &task.RemoteTask{ - Name: "NewETCDNodeHealthCheck", - Desc: "Health check on new etcd", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Prepare: &NodeETCDExist{Not: true}, - Action: new(HealthCheck), - Parallel: true, - Retry: 20, - } - - checkMember := &task.RemoteTask{ - Name: "CheckETCDMember", - Desc: "Check etcd member", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Prepare: &NodeETCDExist{Not: true}, - Action: new(CheckMember), - Parallel: true, - } - - allRefreshETCDConfig := &task.RemoteTask{ - Name: "AllRefreshETCDConfig", - Desc: "Refresh etcd.env config on all etcd", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Action: new(RefreshConfig), - Parallel: false, - } - - allETCDNodeHealthCheck := &task.RemoteTask{ - Name: "AllETCDNodeHealthCheck", - Desc: "Health check on all etcd", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Action: new(HealthCheck), - Parallel: true, - Retry: 20, - } - - tasks := []task.Interface{ - existETCDHealthCheck, - generateETCDConfig, - joinMember, - newETCDNodeHealthCheck, - checkMember, - allRefreshETCDConfig, - allETCDNodeHealthCheck, - } - return tasks -} - -type BackupModule struct { - common.KubeModule - Skip bool -} - -func (p *BackupModule) IsSkip() bool { - return p.Skip -} - -func (b *BackupModule) Init() { - b.Name = "ETCDBackupModule" - b.Desc = "Backup ETCD cluster data" - - backupETCD := &task.RemoteTask{ - Name: "BackupETCD", - Desc: "Backup etcd data regularly", - Hosts: b.Runtime.GetHostsByRole(common.ETCD), - Action: new(BackupETCD), - Parallel: true, - } - - generateBackupETCDService := &task.RemoteTask{ - Name: "GenerateBackupETCDService", - Desc: "Generate backup ETCD service", - Hosts: b.Runtime.GetHostsByRole(common.ETCD), - Action: &action.Template{ - Template: templates.BackupETCDService, - Dst: filepath.Join("/etc/systemd/system/", templates.BackupETCDService.Name()), - Data: util.Data{ - "ScriptPath": filepath.Join(b.KubeConf.Cluster.Etcd.BackupScriptDir, "etcd-backup.sh"), - }, - }, - Parallel: true, - } - - generateBackupETCDTimer := &task.RemoteTask{ - Name: "GenerateBackupETCDTimer", - Desc: "Generate backup ETCD timer", - Hosts: b.Runtime.GetHostsByRole(common.ETCD), - Action: &action.Template{ - Template: templates.BackupETCDTimer, - Dst: filepath.Join("/etc/systemd/system/", templates.BackupETCDTimer.Name()), - Data: util.Data{ - "OnCalendarStr": templates.BackupTimeOnCalendar(b.KubeConf.Cluster.Etcd.BackupPeriod), - }, - }, - Parallel: true, - } - - enable := &task.RemoteTask{ - Name: "EnableBackupETCDService", - Desc: "Enable backup etcd service", - Hosts: b.Runtime.GetHostsByRole(common.ETCD), - Action: new(EnableBackupETCDService), - Parallel: true, - } - - b.Tasks = []task.Interface{ - backupETCD, - generateBackupETCDService, - generateBackupETCDTimer, - enable, - } -} diff --git a/cmd/kk/pkg/etcd/prepares.go b/cmd/kk/pkg/etcd/prepares.go deleted file mode 100644 index aa183dbd2..000000000 --- a/cmd/kk/pkg/etcd/prepares.go +++ /dev/null @@ -1,61 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package etcd - -import ( - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type FirstETCDNode struct { - common.KubePrepare - Not bool -} - -func (f *FirstETCDNode) PreCheck(runtime connector.Runtime) (bool, error) { - v, ok := f.PipelineCache.Get(common.ETCDCluster) - if !ok { - return false, errors.New("get etcd cluster status by pipeline cache failed") - } - cluster := v.(*EtcdCluster) - - if (!cluster.clusterExist && runtime.GetHostsByRole(common.ETCD)[0].GetName() == runtime.RemoteHost().GetName()) || - (cluster.clusterExist && strings.Contains(cluster.peerAddresses[0], runtime.RemoteHost().GetInternalAddress())) { - return !f.Not, nil - } - return f.Not, nil -} - -type NodeETCDExist struct { - common.KubePrepare - Not bool -} - -func (n *NodeETCDExist) PreCheck(runtime connector.Runtime) (bool, error) { - host := runtime.RemoteHost() - if v, ok := host.GetCache().GetMustBool(common.ETCDExist); ok { - if v { - return !n.Not, nil - } - return n.Not, nil - } - return false, errors.New("get etcd node status by host label failed") -} diff --git a/cmd/kk/pkg/etcd/tasks.go b/cmd/kk/pkg/etcd/tasks.go deleted file mode 100644 index 0aeaccdd7..000000000 --- a/cmd/kk/pkg/etcd/tasks.go +++ /dev/null @@ -1,437 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package etcd - -import ( - "fmt" - "path/filepath" - "strings" - - "github.com/pkg/errors" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/etcd/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" -) - -type EtcdNode struct { - NodeName string - EtcdName string - EtcdExist bool -} - -type EtcdCluster struct { - clusterExist bool - accessAddresses string - peerAddresses []string -} - -const ( - LocalCertsDir = "localCertsDir" - CertsFileList = "certsFileList" - - NewCluster = "new" - ExistCluster = "existing" -) - -type GetStatus struct { - common.KubeAction -} - -func (g *GetStatus) Execute(runtime connector.Runtime) error { - exist, err := runtime.GetRunner().FileExist("/etc/etcd.env") - if err != nil { - return err - } - - host := runtime.RemoteHost() - cluster := &EtcdCluster{ - clusterExist: true, - accessAddresses: "", - peerAddresses: []string{}, - } - - if exist { - etcdEnv, err := runtime.GetRunner().SudoCmd("cat /etc/etcd.env | grep ETCD_NAME", true) - if err != nil { - return err - } - - etcdName := etcdEnv[strings.Index(etcdEnv, "=")+1:] - // type: string - host.GetCache().Set(common.ETCDName, etcdName) - // type: bool - host.GetCache().Set(common.ETCDExist, true) - - if v, ok := g.PipelineCache.Get(common.ETCDCluster); ok { - c := v.(*EtcdCluster) - c.peerAddresses = append(c.peerAddresses, fmt.Sprintf("%s=https://%s:2380", etcdName, host.GetInternalAddress())) - c.clusterExist = true - // type: *EtcdCluster - g.PipelineCache.Set(common.ETCDCluster, c) - } else { - cluster.peerAddresses = append(cluster.peerAddresses, fmt.Sprintf("%s=https://%s:2380", etcdName, host.GetInternalAddress())) - cluster.clusterExist = true - g.PipelineCache.Set(common.ETCDCluster, cluster) - } - } else { - host.GetCache().Set(common.ETCDName, fmt.Sprintf("etcd-%s", host.GetName())) - host.GetCache().Set(common.ETCDExist, false) - - if _, ok := g.PipelineCache.Get(common.ETCDCluster); !ok { - cluster.clusterExist = false - g.PipelineCache.Set(common.ETCDCluster, cluster) - } - } - return nil -} - -type SyncCertsFile struct { - common.KubeAction -} - -func (s *SyncCertsFile) Execute(runtime connector.Runtime) error { - localCertsDir, ok := s.ModuleCache.Get(LocalCertsDir) - if !ok { - return errors.New("get etcd local certs dir by module cache failed") - } - files, ok := s.ModuleCache.Get(CertsFileList) - if !ok { - return errors.New("get etcd certs file list by module cache failed") - } - dir := localCertsDir.(string) - fileList := files.([]string) - - for _, fileName := range fileList { - if err := runtime.GetRunner().SudoScp(filepath.Join(dir, fileName), filepath.Join(common.ETCDCertDir, fileName)); err != nil { - return errors.Wrap(errors.WithStack(err), "scp etcd certs file failed") - } - } - - return nil -} - -type InstallETCDBinary struct { - common.KubeAction -} - -func (g *InstallETCDBinary) Execute(runtime connector.Runtime) error { - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - binariesMapObj, ok := g.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) - if !ok { - return errors.New("get KubeBinary by pipeline cache failed") - } - binariesMap := binariesMapObj.(map[string]*files.KubeBinary) - binary, ok := binariesMap["etcd"] - if !ok { - return fmt.Errorf("get kube binary etcd info failed: no such key") - } - - dst := filepath.Join(common.TmpDir, binary.FileName) - if err := runtime.GetRunner().Scp(binary.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), "sync etcd tar.gz failed") - } - - etcdDir := strings.TrimSuffix(binary.FileName, ".tar.gz") - installCmd := fmt.Sprintf("tar -zxf %s && cp -f %s/etcd* /usr/local/bin/ && chmod +x /usr/local/bin/etcd* && rm -rf %s", dst, etcdDir, etcdDir) - if _, err := runtime.GetRunner().SudoCmd(installCmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "install etcd binaries failed") - } - return nil -} - -type GenerateAccessAddress struct { - common.KubeAction -} - -func (g *GenerateAccessAddress) Execute(runtime connector.Runtime) error { - var addrList []string - for _, host := range runtime.GetHostsByRole(common.ETCD) { - addrList = append(addrList, fmt.Sprintf("https://%s:2379", host.GetInternalAddress())) - } - - accessAddresses := strings.Join(addrList, ",") - if v, ok := g.PipelineCache.Get(common.ETCDCluster); ok { - cluster := v.(*EtcdCluster) - cluster.accessAddresses = accessAddresses - g.PipelineCache.Set(common.ETCDCluster, cluster) - } else { - return errors.New("get etcd cluster status by pipeline cache failed") - } - return nil -} - -type HealthCheck struct { - common.KubeAction -} - -func (h *HealthCheck) Execute(runtime connector.Runtime) error { - if v, ok := h.PipelineCache.Get(common.ETCDCluster); ok { - cluster := v.(*EtcdCluster) - if err := healthCheck(runtime, cluster); err != nil { - return err - } - } else { - return errors.New("get etcd cluster status by pipeline cache failed") - } - return nil -} - -func healthCheck(runtime connector.Runtime, cluster *EtcdCluster) error { - host := runtime.RemoteHost() - checkHealthCmd := fmt.Sprintf("export ETCDCTL_API=2;"+ - "export ETCDCTL_CERT_FILE='/etc/ssl/etcd/ssl/admin-%s.pem';"+ - "export ETCDCTL_KEY_FILE='/etc/ssl/etcd/ssl/admin-%s-key.pem';"+ - "export ETCDCTL_CA_FILE='/etc/ssl/etcd/ssl/ca.pem';"+ - "%s/etcdctl --endpoints=%s cluster-health | grep -q 'cluster is healthy'", - host.GetName(), host.GetName(), common.BinDir, cluster.accessAddresses) - if _, err := runtime.GetRunner().SudoCmd(checkHealthCmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "etcd health check failed") - } - return nil -} - -type GenerateConfig struct { - common.KubeAction -} - -func (g *GenerateConfig) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - etcdName, ok := host.GetCache().GetMustString(common.ETCDName) - if !ok { - return errors.New("get etcd node status by host label failed") - } - - if v, ok := g.PipelineCache.Get(common.ETCDCluster); ok { - cluster := v.(*EtcdCluster) - - cluster.peerAddresses = append(cluster.peerAddresses, fmt.Sprintf("%s=https://%s:2380", etcdName, host.GetInternalAddress())) - g.PipelineCache.Set(common.ETCDCluster, cluster) - - if !cluster.clusterExist { - if err := refreshConfig(g.KubeConf, runtime, cluster.peerAddresses, NewCluster, etcdName); err != nil { - return err - } - } else { - if err := refreshConfig(g.KubeConf, runtime, cluster.peerAddresses, ExistCluster, etcdName); err != nil { - return err - } - } - return nil - } else { - return errors.New("get etcd cluster status by pipeline cache failed") - } -} - -type RefreshConfig struct { - common.KubeAction - ToExisting bool -} - -func (r *RefreshConfig) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - etcdName, ok := host.GetCache().GetMustString(common.ETCDName) - if !ok { - return errors.New("get etcd node status by host label failed") - } - - if v, ok := r.PipelineCache.Get(common.ETCDCluster); ok { - cluster := v.(*EtcdCluster) - - if r.ToExisting { - if err := refreshConfig(r.KubeConf, runtime, cluster.peerAddresses, ExistCluster, etcdName); err != nil { - return err - } - return nil - } - - if !cluster.clusterExist { - if err := refreshConfig(r.KubeConf, runtime, cluster.peerAddresses, NewCluster, etcdName); err != nil { - return err - } - } else { - if err := refreshConfig(r.KubeConf, runtime, cluster.peerAddresses, ExistCluster, etcdName); err != nil { - return err - } - } - return nil - } - return errors.New("get etcd cluster status by pipeline cache failed") -} - -func refreshConfig(KubeConf *common.KubeConf, runtime connector.Runtime, endpoints []string, state, etcdName string) error { - host := runtime.RemoteHost() - - UnsupportedArch := false - if host.GetArch() != "amd64" { - UnsupportedArch = true - } - - templateAction := action.Template{ - Template: templates.EtcdEnv, - Dst: filepath.Join("/etc/", templates.EtcdEnv.Name()), - Data: util.Data{ - "Tag": kubekeyapiv1alpha2.DefaultEtcdVersion, - "Name": etcdName, - "Ip": host.GetInternalAddress(), - "Hostname": host.GetName(), - "State": state, - "PeerAddresses": strings.Join(endpoints, ","), - "UnsupportedArch": UnsupportedArch, - "Arch": host.GetArch(), - "DataDir": KubeConf.Cluster.Etcd.DataDir, - "CompactionRetention": KubeConf.Cluster.Etcd.AutoCompactionRetention, - "SnapshotCount": KubeConf.Cluster.Etcd.SnapshotCount, - "Metrics": KubeConf.Cluster.Etcd.Metrics, - "QuotaBackendBytes": KubeConf.Cluster.Etcd.QuotaBackendBytes, - "MaxRequestBytes": KubeConf.Cluster.Etcd.MaxRequestBytes, - "LogLevel": KubeConf.Cluster.Etcd.LogLevel, - "MaxSnapshots": KubeConf.Cluster.Etcd.MaxSnapshots, - "MaxWals": KubeConf.Cluster.Etcd.MaxWals, - "ElectionTimeout": KubeConf.Cluster.Etcd.ElectionTimeout, - "HeartbeatInterval": KubeConf.Cluster.Etcd.HeartbeatInterval, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type JoinMember struct { - common.KubeAction -} - -func (j *JoinMember) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - etcdName, ok := host.GetCache().GetMustString(common.ETCDName) - if !ok { - return errors.New("get etcd node status by host label failed") - } - - if v, ok := j.PipelineCache.Get(common.ETCDCluster); ok { - cluster := v.(*EtcdCluster) - joinMemberCmd := fmt.Sprintf("export ETCDCTL_API=2;"+ - "export ETCDCTL_CERT_FILE='/etc/ssl/etcd/ssl/admin-%s.pem';"+ - "export ETCDCTL_KEY_FILE='/etc/ssl/etcd/ssl/admin-%s-key.pem';"+ - "export ETCDCTL_CA_FILE='/etc/ssl/etcd/ssl/ca.pem';"+ - "%s/etcdctl --endpoints=%s member add %s %s", - host.GetName(), host.GetName(), common.BinDir, cluster.accessAddresses, etcdName, - fmt.Sprintf("https://%s:2380", host.GetInternalAddress())) - - if _, err := runtime.GetRunner().SudoCmd(joinMemberCmd, true); err != nil { - return errors.Wrap(errors.WithStack(err), "add etcd member failed") - } - } else { - return errors.New("get etcd cluster status by pipeline cache failed") - } - - // After adding a new member for etcd, it is necessary to start the new member as the etcd cluster may experience abnormal behavior. - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && systemctl restart etcd && systemctl enable etcd", true); err != nil { - return errors.Wrap(errors.WithStack(err), "start etcd failed") - } - - return nil -} - -type CheckMember struct { - common.KubeAction -} - -func (c *CheckMember) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - if v, ok := c.PipelineCache.Get(common.ETCDCluster); ok { - cluster := v.(*EtcdCluster) - checkMemberCmd := fmt.Sprintf("export ETCDCTL_API=2;"+ - "export ETCDCTL_CERT_FILE='/etc/ssl/etcd/ssl/admin-%s.pem';"+ - "export ETCDCTL_KEY_FILE='/etc/ssl/etcd/ssl/admin-%s-key.pem';"+ - "export ETCDCTL_CA_FILE='/etc/ssl/etcd/ssl/ca.pem';"+ - "%s/etcdctl --no-sync --endpoints=%s member list", host.GetName(), host.GetName(), common.BinDir, cluster.accessAddresses) - memberList, err := runtime.GetRunner().SudoCmd(checkMemberCmd, true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "list etcd member failed") - } - if !strings.Contains(memberList, fmt.Sprintf("https://%s:2379", host.GetInternalAddress())) { - return errors.Wrap(errors.WithStack(err), "add etcd member failed") - } - } else { - return errors.New("get etcd cluster status by pipeline cache failed") - } - return nil -} - -type RestartETCD struct { - common.KubeAction -} - -func (r *RestartETCD) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && systemctl restart etcd && systemctl enable etcd", true); err != nil { - return errors.Wrap(errors.WithStack(err), "start etcd failed") - } - return nil -} - -type BackupETCD struct { - common.KubeAction -} - -func (b *BackupETCD) Execute(runtime connector.Runtime) error { - templateAction := action.Template{ - Template: templates.EtcdBackupScript, - Dst: filepath.Join(b.KubeConf.Cluster.Etcd.BackupScriptDir, "etcd-backup.sh"), - Data: util.Data{ - "Hostname": runtime.RemoteHost().GetName(), - "Etcdendpoint": fmt.Sprintf("https://%s:2379", runtime.RemoteHost().GetInternalAddress()), - "DataDir": b.KubeConf.Cluster.Etcd.DataDir, - "Backupdir": b.KubeConf.Cluster.Etcd.BackupDir, - "KeepbackupNumber": b.KubeConf.Cluster.Etcd.KeepBackupNumber + 1, - "EtcdBackupScriptDir": b.KubeConf.Cluster.Etcd.BackupScriptDir, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chmod +x %s/etcd-backup.sh", b.KubeConf.Cluster.Etcd.BackupScriptDir), false); err != nil { - return errors.Wrap(errors.WithStack(err), "chmod etcd backup script failed") - } - return nil -} - -type EnableBackupETCDService struct { - common.KubeAction -} - -func (e *EnableBackupETCDService) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("systemctl enable --now backup-etcd.timer", - false); err != nil { - return errors.Wrap(errors.WithStack(err), "enable backup-etcd.service failed") - } - return nil -} diff --git a/cmd/kk/pkg/etcd/templates/backup_script.go b/cmd/kk/pkg/etcd/templates/backup_script.go deleted file mode 100644 index 9445aff12..000000000 --- a/cmd/kk/pkg/etcd/templates/backup_script.go +++ /dev/null @@ -1,65 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -// EtcdBackupScriptTmpl defines the template of etcd backup script. -var EtcdBackupScript = template.Must(template.New("etcd-backup.sh").Parse( - dedent.Dedent(`#!/bin/bash - -set -o errexit -set -o nounset -set -o pipefail - -ETCDCTL_PATH='/usr/local/bin/etcdctl' -ENDPOINTS='{{ .Etcdendpoint }}' -{{- if .DataDir }} -ETCD_DATA_DIR="{{ .DataDir }}" -{{- else }} -ETCD_DATA_DIR="/var/lib/etcd" -{{- end }} -BACKUP_DIR="{{ .Backupdir }}/etcd-$(date +%Y-%m-%d-%H-%M-%S)" -KEEPBACKUPNUMBER='{{ .KeepbackupNumber }}' -ETCDBACKUPSCIPT='{{ .EtcdBackupScriptDir }}' - -ETCDCTL_CERT="/etc/ssl/etcd/ssl/admin-{{ .Hostname }}.pem" -ETCDCTL_KEY="/etc/ssl/etcd/ssl/admin-{{ .Hostname }}-key.pem" -ETCDCTL_CA_FILE="/etc/ssl/etcd/ssl/ca.pem" - -[ ! -d $BACKUP_DIR ] && mkdir -p $BACKUP_DIR - -export ETCDCTL_API=2;$ETCDCTL_PATH backup --data-dir $ETCD_DATA_DIR --backup-dir $BACKUP_DIR - -sleep 3 - -{ -export ETCDCTL_API=3;$ETCDCTL_PATH --endpoints="$ENDPOINTS" snapshot save $BACKUP_DIR/snapshot.db \ - --cacert="$ETCDCTL_CA_FILE" \ - --cert="$ETCDCTL_CERT" \ - --key="$ETCDCTL_KEY" -} > /dev/null - -sleep 3 - -cd $BACKUP_DIR/../ && ls -lt |awk '{if(NR > '$KEEPBACKUPNUMBER'){print "rm -rf "$9}}'|sh - -`))) diff --git a/cmd/kk/pkg/etcd/templates/backup_service.go b/cmd/kk/pkg/etcd/templates/backup_service.go deleted file mode 100644 index d2b25ddb9..000000000 --- a/cmd/kk/pkg/etcd/templates/backup_service.go +++ /dev/null @@ -1,71 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "fmt" - "strconv" - "text/template" - - "github.com/lithammer/dedent" -) - -var ( - // BackupETCDService defines the template of backup-etcd service for systemd. - BackupETCDService = template.Must(template.New("backup-etcd.service").Parse( - dedent.Dedent(`[Unit] -Description=Backup ETCD -[Service] -Type=oneshot -ExecStart={{ .ScriptPath }} - `))) - - // BackupETCDTimer defines the template of backup-etcd timer for systemd. - BackupETCDTimer = template.Must(template.New("backup-etcd.timer").Parse( - dedent.Dedent(`[Unit] -Description=Timer to backup ETCD -[Timer] -{{- if .OnCalendarStr }} -OnCalendar={{ .OnCalendarStr }} -{{- else }} -OnCalendar=*-*-* 02:00:00 -{{- end }} -Unit=backup-etcd.service -[Install] -WantedBy=multi-user.target - `))) -) - -func BackupTimeOnCalendar(period int) string { - var onCalendar string - if period <= 0 { - onCalendar = "*-*-* *:00/30:00" - } else if period < 60 { - onCalendar = fmt.Sprintf("*-*-* *:00/%d:00", period) - } else if period >= 60 && period < 1440 { - hour := strconv.Itoa(period / 60) - minute := strconv.Itoa(period % 60) - if period%60 == 0 { - onCalendar = fmt.Sprintf("*-*-* 00/%s:00:00", hour) - } else { - onCalendar = fmt.Sprintf("*-*-* 00/%s:%s:00", hour, minute) - } - } else { - onCalendar = "*-*-* 02:00:00" - } - return onCalendar -} diff --git a/cmd/kk/pkg/etcd/templates/backup_service_test.go b/cmd/kk/pkg/etcd/templates/backup_service_test.go deleted file mode 100644 index 4abeba215..000000000 --- a/cmd/kk/pkg/etcd/templates/backup_service_test.go +++ /dev/null @@ -1,52 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "testing" -) - -func TestBackupTimeOnCalendar(t *testing.T) { - tests := []struct { - period int - want string - }{ - { - 30, - "*-*-* *:00/30:00", - }, - { - 60, - "*-*-* 00/1:00:00", - }, - { - 70, - "*-*-* 00/1:10:00", - }, - { - 1500, - "*-*-* 00:00:00", - }, - } - for _, tt := range tests { - t.Run("", func(t *testing.T) { - if got := BackupTimeOnCalendar(tt.period); got != tt.want { - t.Errorf("BackupTimeOnCalendar() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/cmd/kk/pkg/etcd/templates/etcd_env.go b/cmd/kk/pkg/etcd/templates/etcd_env.go deleted file mode 100644 index f6506c766..000000000 --- a/cmd/kk/pkg/etcd/templates/etcd_env.go +++ /dev/null @@ -1,101 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -// EtcdEnv defines the template of etcd's env. -var EtcdEnv = template.Must(template.New("etcd.env").Parse( - dedent.Dedent(`# Environment file for etcd {{ .Tag }} -{{- if .DataDir }} -ETCD_DATA_DIR={{ .DataDir }} -{{- else }} -ETCD_DATA_DIR=/var/lib/etcd -{{- end }} -ETCD_ADVERTISE_CLIENT_URLS=https://{{ .Ip }}:2379 -ETCD_INITIAL_ADVERTISE_PEER_URLS=https://{{ .Ip }}:2380 -ETCD_INITIAL_CLUSTER_STATE={{ .State }} -ETCD_METRICS=basic -ETCD_LISTEN_CLIENT_URLS=https://{{ .Ip }}:2379,https://127.0.0.1:2379 -ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd -ETCD_LISTEN_PEER_URLS=https://{{ .Ip }}:2380 -ETCD_NAME={{ .Name }} -ETCD_PROXY=off -ETCD_ENABLE_V2=true -ETCD_INITIAL_CLUSTER={{ .PeerAddresses }} -{{- if .ElectionTimeout }} -ETCD_ELECTION_TIMEOUT={{ .ElectionTimeout }} -{{- else }} -ETCD_ELECTION_TIMEOUT=5000 -{{- end }} -{{- if .HeartbeatInterval }} -ETCD_HEARTBEAT_INTERVAL={{ .HeartbeatInterval }} -{{- else }} -ETCD_HEARTBEAT_INTERVAL=250 -{{- end }} -{{- if .CompactionRetention }} -ETCD_AUTO_COMPACTION_RETENTION={{ .CompactionRetention }} -{{- else }} -ETCD_AUTO_COMPACTION_RETENTION=8 -{{- end }} -{{- if .SnapshotCount }} -ETCD_SNAPSHOT_COUNT={{ .SnapshotCount }} -{{- else }} -ETCD_SNAPSHOT_COUNT=10000 -{{- end }} -{{- if .Metrics }} -ETCD_METRICS={{ .Metrics }} -{{- end }} -{{- if .QuotaBackendBytes }} -ETCD_QUOTA_BACKEND_BYTES={{ .QuotaBackendBytes }} -{{- end }} -{{- if .MaxRequestBytes }} -ETCD_MAX_REQUEST_BYTES={{ .MaxRequestBytes }} -{{- end }} -{{- if .MaxSnapshots }} -ETCD_MAX_SNAPSHOTS={{ .MaxSnapshots }} -{{- end }} -{{- if .MaxWals }} -ETCD_MAX_WALS={{ .MaxWals }} -{{- end }} -{{- if .LogLevel }} -ETCD_LOG_LEVEL={{ .LogLevel }} -{{- end }} -{{- if .UnsupportedArch }} -ETCD_UNSUPPORTED_ARCH={{ .Arch }} -{{ end }} - -# TLS settings -ETCD_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.pem -ETCD_CERT_FILE=/etc/ssl/etcd/ssl/member-{{ .Hostname }}.pem -ETCD_KEY_FILE=/etc/ssl/etcd/ssl/member-{{ .Hostname }}-key.pem -ETCD_CLIENT_CERT_AUTH=true - -ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.pem -ETCD_PEER_CERT_FILE=/etc/ssl/etcd/ssl/member-{{ .Hostname }}.pem -ETCD_PEER_KEY_FILE=/etc/ssl/etcd/ssl/member-{{ .Hostname }}-key.pem -ETCD_PEER_CLIENT_CERT_AUTH=True - -# CLI settings -ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 -ETCDCTL_CACERT=/etc/ssl/etcd/ssl/ca.pem -ETCDCTL_KEY=/etc/ssl/etcd/ssl/admin-{{ .Hostname }}-key.pem -ETCDCTL_CERT=/etc/ssl/etcd/ssl/admin-{{ .Hostname }}.pem - `))) diff --git a/cmd/kk/pkg/etcd/templates/etcd_service.go b/cmd/kk/pkg/etcd/templates/etcd_service.go deleted file mode 100644 index 78f2b788c..000000000 --- a/cmd/kk/pkg/etcd/templates/etcd_service.go +++ /dev/null @@ -1,44 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var ( - // ETCDService defines the template of etcd's service for systemd. - ETCDService = template.Must(template.New("etcd.service").Parse( - dedent.Dedent(`[Unit] -Description=etcd -After=network.target - -[Service] -User=root -Type=notify -EnvironmentFile=/etc/etcd.env -ExecStart=/usr/local/bin/etcd -NotifyAccess=all -RestartSec=10s -LimitNOFILE=40000 -Restart=always - -[Install] -WantedBy=multi-user.target - `))) -) diff --git a/cmd/kk/pkg/files/file.go b/cmd/kk/pkg/files/file.go deleted file mode 100644 index bba319b6c..000000000 --- a/cmd/kk/pkg/files/file.go +++ /dev/null @@ -1,328 +0,0 @@ -/* - Copyright 2020 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package files - -import ( - "crypto/sha256" - "fmt" - "io" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/version" -) - -const ( - kubeadm = "kubeadm" - kubelet = "kubelet" - kubectl = "kubectl" - kubecni = "kubecni" - etcd = "etcd" - helm = "helm" - amd64 = "amd64" - arm64 = "arm64" - k3s = "k3s" - k8e = "k8e" - docker = "docker" - crictl = "crictl" - registry = "registry" - harbor = "harbor" - compose = "compose" - containerd = "containerd" - runc = "runc" - calicoctl = "calicoctl" -) - -// KubeBinary Type field const -const ( - CNI = "cni" - CRICTL = "crictl" - DOCKER = "docker" - ETCD = "etcd" - HELM = "helm" - KUBE = "kube" - REGISTRY = "registry" - CONTAINERD = "containerd" - RUNC = "runc" -) - -var ( - // FileSha256 is a hash table the storage the checksum of the binary files. It is parsed from 'version/components.json'. - FileSha256 = map[string]map[string]map[string]string{} -) - -func init() { - FileSha256, _ = version.ParseFilesSha256(version.Components) -} - -type KubeBinary struct { - Type string - ID string - FileName string - Arch string - Version string - Url string - BaseDir string - Zone string - getCmd func(path, url string) string -} - -func NewKubeBinary(name, arch, version, prePath string, getCmd func(path, url string) string) *KubeBinary { - component := new(KubeBinary) - component.ID = name - component.Arch = arch - component.Version = version - component.Zone = os.Getenv("KKZONE") - component.getCmd = getCmd - - switch name { - case etcd: - component.Type = ETCD - component.FileName = fmt.Sprintf("etcd-%s-linux-%s.tar.gz", version, arch) - component.Url = fmt.Sprintf("https://github.com/coreos/etcd/releases/download/%s/etcd-%s-linux-%s.tar.gz", version, version, arch) - if component.Zone == "cn" { - component.Url = fmt.Sprintf( - "https://kubernetes-release.pek3b.qingstor.com/etcd/release/download/%s/etcd-%s-linux-%s.tar.gz", - component.Version, component.Version, component.Arch) - } - case kubeadm: - component.Type = KUBE - component.FileName = kubeadm - component.Url = fmt.Sprintf("https://storage.googleapis.com/kubernetes-release/release/%s/bin/linux/%s/kubeadm", version, arch) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/release/%s/bin/linux/%s/kubeadm", version, arch) - } - case kubelet: - component.Type = KUBE - component.FileName = kubelet - component.Url = fmt.Sprintf("https://storage.googleapis.com/kubernetes-release/release/%s/bin/linux/%s/kubelet", version, arch) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/release/%s/bin/linux/%s/kubelet", version, arch) - } - case kubectl: - component.Type = KUBE - component.FileName = kubectl - component.Url = fmt.Sprintf("https://storage.googleapis.com/kubernetes-release/release/%s/bin/linux/%s/kubectl", version, arch) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/release/%s/bin/linux/%s/kubectl", version, arch) - } - case kubecni: - component.Type = CNI - component.FileName = fmt.Sprintf("cni-plugins-linux-%s-%s.tgz", arch, version) - component.Url = fmt.Sprintf("https://github.com/containernetworking/plugins/releases/download/%s/cni-plugins-linux-%s-%s.tgz", version, arch, version) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://containernetworking.pek3b.qingstor.com/plugins/releases/download/%s/cni-plugins-linux-%s-%s.tgz", version, arch, version) - } - case helm: - component.Type = HELM - component.FileName = helm - component.Url = fmt.Sprintf("https://get.helm.sh/helm-%s-linux-%s.tar.gz", version, arch) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://kubernetes-helm.pek3b.qingstor.com/linux-%s/%s/helm", arch, version) - } - case docker: - component.Type = DOCKER - component.FileName = fmt.Sprintf("docker-%s.tgz", version) - component.Url = fmt.Sprintf("https://download.docker.com/linux/static/stable/%s/docker-%s.tgz", util.ArchAlias(arch), version) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://mirrors.aliyun.com/docker-ce/linux/static/stable/%s/docker-%s.tgz", util.ArchAlias(arch), version) - } - case crictl: - component.Type = CRICTL - component.FileName = fmt.Sprintf("crictl-%s-linux-%s.tar.gz", version, arch) - component.Url = fmt.Sprintf("https://github.com/kubernetes-sigs/cri-tools/releases/download/%s/crictl-%s-linux-%s.tar.gz", version, version, arch) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/cri-tools/releases/download/%s/crictl-%s-linux-%s.tar.gz", version, version, arch) - } - case k3s: - component.Type = KUBE - component.FileName = k3s - component.Url = fmt.Sprintf("https://github.com/k3s-io/k3s/releases/download/%s+k3s1/k3s", version) - if arch == arm64 { - component.Url = fmt.Sprintf("https://github.com/k3s-io/k3s/releases/download/%s+k3s1/k3s-%s", version, arch) - } - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/k3s/releases/download/%s+k3s1/linux/%s/k3s", version, arch) - } - case k8e: - component.Type = KUBE - component.FileName = k8e - component.Url = fmt.Sprintf("https://github.com/xiaods/k8e/releases/download/%s+k8e2/k8e", version) - if arch == arm64 { - component.Url = fmt.Sprintf("https://github.com/xiaods/k8e/releases/download/%s+k8e2/k8e-%s", version, arch) - } - case registry: - component.Type = REGISTRY - component.FileName = fmt.Sprintf("registry-%s-linux-%s.tar.gz", version, arch) - component.Url = fmt.Sprintf("https://github.com/kubesphere/kubekey/releases/download/v2.0.0-alpha.1/registry-%s-linux-%s.tar.gz", version, arch) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/registry/%s/registry-%s-linux-%s.tar.gz", version, version, arch) - } - component.BaseDir = filepath.Join(prePath, component.Type, component.ID, component.Version, component.Arch) - case harbor: - component.Type = REGISTRY - component.FileName = fmt.Sprintf("harbor-offline-installer-%s.tgz", version) - component.Url = fmt.Sprintf("https://github.com/goharbor/harbor/releases/download/%s/harbor-offline-installer-%s.tgz", version, version) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/harbor/releases/download/%s/harbor-offline-installer-%s.tgz", version, version) - } - component.BaseDir = filepath.Join(prePath, component.Type, component.ID, component.Version, component.Arch) - case compose: - component.Type = REGISTRY - component.FileName = "docker-compose-linux-x86_64" - component.Url = fmt.Sprintf("https://github.com/docker/compose/releases/download/%s/docker-compose-linux-%s", version, util.ArchAlias(arch)) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/docker/compose/releases/download/%s/docker-compose-linux-%s", version, util.ArchAlias(arch)) - } - component.BaseDir = filepath.Join(prePath, component.Type, component.ID, component.Version, component.Arch) - case containerd: - component.Type = CONTAINERD - component.FileName = fmt.Sprintf("containerd-%s-linux-%s.tar.gz", version, arch) - component.Url = fmt.Sprintf("https://github.com/containerd/containerd/releases/download/v%s/containerd-%s-linux-%s.tar.gz", version, version, arch) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/containerd/containerd/releases/download/v%s/containerd-%s-linux-%s.tar.gz", version, version, arch) - } - case runc: - component.Type = RUNC - component.FileName = fmt.Sprintf("runc.%s", arch) - component.Url = fmt.Sprintf("https://github.com/opencontainers/runc/releases/download/%s/runc.%s", version, arch) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/opencontainers/runc/releases/download/%s/runc.%s", version, arch) - } - case calicoctl: - component.Type = CNI - component.FileName = calicoctl - component.Url = fmt.Sprintf("https://github.com/projectcalico/calico/releases/download/%s/calicoctl-linux-%s", version, arch) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/projectcalico/calico/releases/download/%s/calicoctl-linux-%s", version, arch) - } - default: - logger.Log.Fatalf("unsupported kube binaries %s", name) - } - - if component.BaseDir == "" { - component.BaseDir = filepath.Join(prePath, component.Type, component.Version, component.Arch) - } - - return component -} - -func (b *KubeBinary) CreateBaseDir() error { - if err := util.CreateDir(b.BaseDir); err != nil { - return err - } - return nil -} - -func (b *KubeBinary) Path() string { - return filepath.Join(b.BaseDir, b.FileName) -} - -func (b *KubeBinary) GetCmd() string { - cmd := b.getCmd(b.Path(), b.Url) - - if b.ID == helm && b.Zone != "cn" { - get := b.getCmd(filepath.Join(b.BaseDir, fmt.Sprintf("helm-%s-linux-%s.tar.gz", b.Version, b.Arch)), b.Url) - cmd = fmt.Sprintf("%s && cd %s && tar -zxf helm-%s-linux-%s.tar.gz && mv linux-%s/helm . && rm -rf *linux-%s*", - get, b.BaseDir, b.Version, b.Arch, b.Arch, b.Arch) - } - return cmd -} - -func (b *KubeBinary) GetSha256() string { - s := FileSha256[b.ID][b.Arch][b.Version] - return s -} - -func (b *KubeBinary) Download() error { - for i := 5; i > 0; i-- { - cmd := exec.Command("/bin/sh", "-c", b.GetCmd()) - stdout, err := cmd.StdoutPipe() - if err != nil { - return err - } - cmd.Stderr = cmd.Stdout - - if err = cmd.Start(); err != nil { - return err - } - for { - tmp := make([]byte, 1024) - _, err := stdout.Read(tmp) - fmt.Print(string(tmp)) // Get the output from the pipeline in real time and print it to the terminal - if errors.Is(err, io.EOF) { - break - } else if err != nil { - logger.Log.Errorln(err) - break - } - } - if err = cmd.Wait(); err != nil { - if os.Getenv("KKZONE") != "cn" { - logger.Log.Warningln("Having a problem with accessing https://storage.googleapis.com? You can try again after setting environment 'export KKZONE=cn'") - } - return err - } - - if err := b.SHA256Check(); err != nil { - if i == 1 { - return err - } - path := b.Path() - _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", path)).Run() - continue - } - break - } - return nil -} - -// SHA256Check is used to hash checks on downloaded binary. (sha256) -func (b *KubeBinary) SHA256Check() error { - output, err := sha256sum(b.Path()) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("Failed to check SHA256 of %s", b.Path())) - } - - if strings.TrimSpace(b.GetSha256()) == "" { - return errors.New(fmt.Sprintf("No SHA256 found for %s. %s is not supported.", b.ID, b.Version)) - } - if output != b.GetSha256() { - return errors.New(fmt.Sprintf("SHA256 no match. %s not equal %s", b.GetSha256(), output)) - } - return nil -} - -func sha256sum(path string) (string, error) { - file, err := os.Open(path) - if err != nil { - return "", err - } - defer file.Close() - - hasher := sha256.New() - if _, err := io.Copy(hasher, file); err != nil { - return "", err - } - return fmt.Sprintf("%x", hasher.Sum(nil)), nil -} diff --git a/cmd/kk/pkg/filesystem/module.go b/cmd/kk/pkg/filesystem/module.go deleted file mode 100644 index fa0dacbe4..000000000 --- a/cmd/kk/pkg/filesystem/module.go +++ /dev/null @@ -1,82 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package filesystem - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" -) - -type ChownModule struct { - module.BaseTaskModule -} - -func (c *ChownModule) Init() { - c.Name = "ChownModule" - c.Desc = "Change file and dir mode and owner" - - userKubeDir := &task.RemoteTask{ - Name: "ChownFileAndDir", - Desc: "Chown user $HOME/.kube dir", - Hosts: c.Runtime.GetHostsByRole(common.K8s), - Action: &ChownFileAndDir{Path: "$HOME/.kube"}, - Parallel: true, - } - - c.Tasks = []task.Interface{ - userKubeDir, - } -} - -type ChownWorkDirModule struct { - module.BaseTaskModule -} - -func (c *ChownWorkDirModule) Init() { - c.Name = "ChownWorkerModule" - c.Desc = "Change kubekey work dir mode and owner" - - userKubeDir := &task.LocalTask{ - Name: "ChownFileAndDir", - Desc: "Chown ./kubekey dir", - Action: &LocalTaskChown{Path: c.Runtime.GetWorkDir()}, - } - - c.Tasks = []task.Interface{ - userKubeDir, - } -} - -type ChownOutputModule struct { - common.ArtifactModule -} - -func (c *ChownOutputModule) Init() { - c.Name = "ChownOutputModule" - c.Desc = "Change file and dir owner" - - output := &task.LocalTask{ - Name: "Chown output file", - Desc: "Chown output file", - Action: &LocalTaskChown{Path: c.Manifest.Arg.Output}, - } - - c.Tasks = []task.Interface{ - output, - } -} diff --git a/cmd/kk/pkg/filesystem/task.go b/cmd/kk/pkg/filesystem/task.go deleted file mode 100644 index b824e7825..000000000 --- a/cmd/kk/pkg/filesystem/task.go +++ /dev/null @@ -1,72 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package filesystem - -import ( - "fmt" - "os/exec" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -type ChownFileAndDir struct { - action.BaseAction - Path string -} - -func (c *ChownFileAndDir) Execute(runtime connector.Runtime) error { - exist, err := runtime.GetRunner().FileExist(c.Path) - if err != nil { - return errors.Wrapf(errors.WithStack(err), "get user %s failed", c.Path) - } - - if exist { - userId, err := runtime.GetRunner().Cmd("echo $(id -u)", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get user id failed") - } - - userGroupId, err := runtime.GetRunner().Cmd("echo $(id -g)", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get user group id failed") - } - - chownKubeConfig := fmt.Sprintf("chown -R %s:%s %s", userId, userGroupId, c.Path) - if _, err := runtime.GetRunner().SudoCmd(chownKubeConfig, false); err != nil { - return errors.Wrapf(errors.WithStack(err), "chown user %s failed", c.Path) - } - } - return nil -} - -type LocalTaskChown struct { - action.BaseAction - Path string -} - -func (l *LocalTaskChown) Execute(runtime connector.Runtime) error { - if exist := util.IsExist(l.Path); exist { - if err := exec.Command("/bin/sh", "-c", fmt.Sprintf("chown -R ${SUDO_UID}:${SUDO_GID} %s", l.Path)).Run(); err != nil { - return errors.Wrapf(errors.WithStack(err), "chown %s failed", l.Path) - } - } - return nil -} diff --git a/cmd/kk/pkg/images/copy.go b/cmd/kk/pkg/images/copy.go deleted file mode 100644 index 4d4cb2e56..000000000 --- a/cmd/kk/pkg/images/copy.go +++ /dev/null @@ -1,86 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package images - -import ( - "context" - "os" - - "github.com/containers/image/v5/copy" - "github.com/containers/image/v5/signature" - "github.com/containers/image/v5/transports/alltransports" -) - -type CopyImageOptions struct { - srcImage *srcImageOptions - destImage *destImageOptions - imageListSelection copy.ImageListSelection -} - -func (c *CopyImageOptions) Copy() error { - policyContext, err := getPolicyContext() - if err != nil { - return err - } - defer policyContext.Destroy() - - srcRef, err := alltransports.ParseImageName(c.srcImage.imageName) - if err != nil { - return err - } - destRef, err := alltransports.ParseImageName(c.destImage.imageName) - if err != nil { - return err - } - - srcContext := c.srcImage.systemContext() - destContext := c.destImage.systemContext() - - _, err = copy.Image(context.Background(), policyContext, destRef, srcRef, ©.Options{ - ReportWriter: os.Stdout, - SourceCtx: srcContext, - DestinationCtx: destContext, - ImageListSelection: c.imageListSelection, - }) - if err != nil { - return err - } - return nil -} - -func getPolicyContext() (*signature.PolicyContext, error) { - policy := &signature.Policy{Default: []signature.PolicyRequirement{signature.NewPRInsecureAcceptAnything()}} - return signature.NewPolicyContext(policy) -} - -type Index struct { - Manifests []Manifest -} - -type Manifest struct { - Annotations annotations -} - -type annotations struct { - RefName string `json:"org.opencontainers.image.ref.name"` -} - -func NewIndex() *Index { - return &Index{ - Manifests: []Manifest{}, - } -} diff --git a/cmd/kk/pkg/images/images.go b/cmd/kk/pkg/images/images.go deleted file mode 100644 index f314cbea9..000000000 --- a/cmd/kk/pkg/images/images.go +++ /dev/null @@ -1,151 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package images - -import ( - "fmt" - "os" - - "github.com/pkg/errors" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" -) - -const ( - cnRegistry = "registry.cn-beijing.aliyuncs.com" - cnNamespaceOverride = "kubesphereio" -) - -// Image defines image's info. -type Image struct { - RepoAddr string - Namespace string - NamespaceOverride string - Repo string - Tag string - Group string - Enable bool -} - -// Images contains a list of Image -type Images struct { - Images []Image -} - -// ImageName is used to generate image's full name. -func (image Image) ImageName() string { - return fmt.Sprintf("%s:%s", image.ImageRepo(), image.Tag) -} - -// ImageNamespace is used to get image's namespace -func (image Image) ImageNamespace() string { - if os.Getenv("KKZONE") == "cn" { - if image.RepoAddr == "" || image.RepoAddr == cnRegistry { - image.NamespaceOverride = cnNamespaceOverride - } - } - - if image.NamespaceOverride != "" { - return image.NamespaceOverride - } else { - return image.Namespace - } -} - -// ImageRegistryAddr is used to get image's registry address. -func (image Image) ImageRegistryAddr() string { - if os.Getenv("KKZONE") == "cn" { - if image.RepoAddr == "" || image.RepoAddr == cnRegistry { - image.RepoAddr = cnRegistry - } - } - if image.RepoAddr != "" { - return image.RepoAddr - } else { - return "docker.io" - } -} - -// ImageRepo is used to generate image's repo address. -func (image Image) ImageRepo() string { - var prefix string - - if os.Getenv("KKZONE") == "cn" { - if image.RepoAddr == "" || image.RepoAddr == cnRegistry { - image.RepoAddr = cnRegistry - image.NamespaceOverride = cnNamespaceOverride - } - } - - if image.RepoAddr == "" { - if image.Namespace == "" { - prefix = "" - } else { - prefix = fmt.Sprintf("%s/", image.Namespace) - } - } else { - if image.NamespaceOverride == "" { - if image.Namespace == "" { - prefix = fmt.Sprintf("%s/library/", image.RepoAddr) - } else { - prefix = fmt.Sprintf("%s/%s/", image.RepoAddr, image.Namespace) - } - } else { - prefix = fmt.Sprintf("%s/%s/", image.RepoAddr, image.NamespaceOverride) - } - } - - return fmt.Sprintf("%s%s", prefix, image.Repo) -} - -// PullImages is used to pull images in the list of Image. -func (images *Images) PullImages(runtime connector.Runtime, kubeConf *common.KubeConf) error { - pullCmd := "docker" - switch kubeConf.Cluster.Kubernetes.ContainerManager { - case "crio": - pullCmd = "crictl" - case "containerd": - pullCmd = "crictl" - case "isula": - pullCmd = "isula" - default: - pullCmd = "docker" - } - - host := runtime.RemoteHost() - - for _, image := range images.Images { - switch { - case host.IsRole(common.Master) && image.Group == kubekeyapiv1alpha2.Master && image.Enable, - host.IsRole(common.Worker) && image.Group == kubekeyapiv1alpha2.Worker && image.Enable, - (host.IsRole(common.Master) || host.IsRole(common.Worker)) && image.Group == kubekeyapiv1alpha2.K8s && image.Enable, - host.IsRole(common.ETCD) && image.Group == kubekeyapiv1alpha2.Etcd && image.Enable: - - logger.Log.Messagef(host.GetName(), "downloading image: %s", image.ImageName()) - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("env PATH=$PATH %s pull %s --platform %s", pullCmd, image.ImageName(), host.GetArch()), false); err != nil { - return errors.Wrap(err, "pull image failed") - } - default: - continue - } - - } - return nil -} diff --git a/cmd/kk/pkg/images/module.go b/cmd/kk/pkg/images/module.go deleted file mode 100644 index cc4843e02..000000000 --- a/cmd/kk/pkg/images/module.go +++ /dev/null @@ -1,99 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package images - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" -) - -type PullModule struct { - common.KubeModule - Skip bool -} - -func (p *PullModule) IsSkip() bool { - return p.Skip -} - -func (p *PullModule) Init() { - p.Name = "PullModule" - p.Desc = "Pull images on all nodes" - - pull := &task.RemoteTask{ - Name: "PullImages", - Desc: "Start to pull images on all nodes", - Hosts: p.Runtime.GetAllHosts(), - Action: new(PullImage), - Parallel: true, - } - - p.Tasks = []task.Interface{ - pull, - } -} - -type CopyImagesToLocalModule struct { - common.ArtifactModule -} - -func (c *CopyImagesToLocalModule) Init() { - c.Name = "CopyImagesToLocalModule" - c.Desc = "Copy images to a local OCI path from registries" - - copyImage := &task.LocalTask{ - Name: "SaveImages", - Desc: "Copy images to a local OCI path from registries", - Action: new(SaveImages), - } - - c.Tasks = []task.Interface{ - copyImage, - } -} - -type CopyImagesToRegistryModule struct { - common.KubeModule - Skip bool - ImagePath string -} - -func (c *CopyImagesToRegistryModule) IsSkip() bool { - return c.Skip -} - -func (c *CopyImagesToRegistryModule) Init() { - c.Name = "CopyImagesToRegistryModule" - c.Desc = "Copy images to a private registry from an artifact OCI path" - - copyImage := &task.LocalTask{ - Name: "CopyImagesToRegistry", - Desc: "Copy images to a private registry from an artifact OCI Path", - Action: &CopyImagesToRegistry{ImagesPath: c.ImagePath}, - } - - pushManifest := &task.LocalTask{ - Name: "PushManifest", - Desc: "Push multi-arch manifest to private registry", - Action: new(PushManifest), - } - - c.Tasks = []task.Interface{ - copyImage, - pushManifest, - } -} diff --git a/cmd/kk/pkg/images/tasks.go b/cmd/kk/pkg/images/tasks.go deleted file mode 100644 index f8d59f8d0..000000000 --- a/cmd/kk/pkg/images/tasks.go +++ /dev/null @@ -1,371 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package images - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "reflect" - "strings" - - manifestregistry "github.com/estesp/manifest-tool/v2/pkg/registry" - manifesttypes "github.com/estesp/manifest-tool/v2/pkg/types" - "github.com/pkg/errors" - versionutil "k8s.io/apimachinery/pkg/util/version" - - kubekeyv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - coreutil "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/registry" -) - -type PullImage struct { - common.KubeAction -} - -func (p *PullImage) Execute(runtime connector.Runtime) error { - if !p.KubeConf.Arg.SkipPullImages { - i := Images{} - i.Images = []Image{ - GetImage(runtime, p.KubeConf, "etcd"), - GetImage(runtime, p.KubeConf, "pause"), - GetImage(runtime, p.KubeConf, "kube-apiserver"), - GetImage(runtime, p.KubeConf, "kube-controller-manager"), - GetImage(runtime, p.KubeConf, "kube-scheduler"), - GetImage(runtime, p.KubeConf, "kube-proxy"), - GetImage(runtime, p.KubeConf, "coredns"), - GetImage(runtime, p.KubeConf, "k8s-dns-node-cache"), - GetImage(runtime, p.KubeConf, "calico-kube-controllers"), - GetImage(runtime, p.KubeConf, "calico-cni"), - GetImage(runtime, p.KubeConf, "calico-node"), - GetImage(runtime, p.KubeConf, "calico-flexvol"), - GetImage(runtime, p.KubeConf, "cilium"), - GetImage(runtime, p.KubeConf, "cilium-operator-generic"), - GetImage(runtime, p.KubeConf, "flannel"), - GetImage(runtime, p.KubeConf, "flannel-cni-plugin"), - GetImage(runtime, p.KubeConf, "kubeovn"), - GetImage(runtime, p.KubeConf, "haproxy"), - GetImage(runtime, p.KubeConf, "kubevip"), - } - - if err := i.PullImages(runtime, p.KubeConf); err != nil { - return err - } - } - return nil -} - -// GetImage defines the list of all images and gets image object by name. -func GetImage(runtime connector.ModuleRuntime, kubeConf *common.KubeConf, name string) Image { - var image Image - pauseTag, corednsTag := "3.2", "1.6.9" - - if versionutil.MustParseSemantic(kubeConf.Cluster.Kubernetes.Version).LessThan(versionutil.MustParseSemantic("v1.21.0")) { - pauseTag = "3.2" - corednsTag = "1.6.9" - } - if versionutil.MustParseSemantic(kubeConf.Cluster.Kubernetes.Version).AtLeast(versionutil.MustParseSemantic("v1.21.0")) || - (kubeConf.Cluster.Kubernetes.ContainerManager != "" && kubeConf.Cluster.Kubernetes.ContainerManager != "docker") { - pauseTag = "3.4.1" - corednsTag = "1.8.0" - } - if versionutil.MustParseSemantic(kubeConf.Cluster.Kubernetes.Version).AtLeast(versionutil.MustParseSemantic("v1.22.0")) { - pauseTag = "3.5" - corednsTag = "1.8.0" - } - if versionutil.MustParseSemantic(kubeConf.Cluster.Kubernetes.Version).AtLeast(versionutil.MustParseSemantic("v1.23.0")) { - pauseTag = "3.6" - corednsTag = "1.8.6" - } - if versionutil.MustParseSemantic(kubeConf.Cluster.Kubernetes.Version).AtLeast(versionutil.MustParseSemantic("v1.24.0")) { - pauseTag = "3.7" - corednsTag = "1.8.6" - } - if versionutil.MustParseSemantic(kubeConf.Cluster.Kubernetes.Version).AtLeast(versionutil.MustParseSemantic("v1.25.0")) { - pauseTag = "3.8" - corednsTag = "1.9.3" - } - - logger.Log.Debugf("pauseTag: %s, corednsTag: %s", pauseTag, corednsTag) - - ImageList := map[string]Image{ - "pause": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "pause", Tag: pauseTag, Group: kubekeyv1alpha2.K8s, Enable: true}, - "etcd": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "etcd", Tag: kubekeyv1alpha2.DefaultEtcdVersion, Group: kubekeyv1alpha2.Master, Enable: strings.EqualFold(kubeConf.Cluster.Etcd.Type, kubekeyv1alpha2.Kubeadm)}, - "kube-apiserver": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "kube-apiserver", Tag: kubeConf.Cluster.Kubernetes.Version, Group: kubekeyv1alpha2.Master, Enable: true}, - "kube-controller-manager": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "kube-controller-manager", Tag: kubeConf.Cluster.Kubernetes.Version, Group: kubekeyv1alpha2.Master, Enable: true}, - "kube-scheduler": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "kube-scheduler", Tag: kubeConf.Cluster.Kubernetes.Version, Group: kubekeyv1alpha2.Master, Enable: true}, - "kube-proxy": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "kube-proxy", Tag: kubeConf.Cluster.Kubernetes.Version, Group: kubekeyv1alpha2.K8s, Enable: !kubeConf.Cluster.Kubernetes.DisableKubeProxy}, - - // network - "coredns": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "coredns", Repo: "coredns", Tag: corednsTag, Group: kubekeyv1alpha2.K8s, Enable: true}, - "k8s-dns-node-cache": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "k8s-dns-node-cache", Tag: "1.22.20", Group: kubekeyv1alpha2.K8s, Enable: kubeConf.Cluster.Kubernetes.EnableNodelocaldns()}, - "calico-kube-controllers": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "calico", Repo: "kube-controllers", Tag: kubekeyv1alpha2.DefaultCalicoVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "calico")}, - "calico-cni": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "calico", Repo: "cni", Tag: kubekeyv1alpha2.DefaultCalicoVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "calico")}, - "calico-node": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "calico", Repo: "node", Tag: kubekeyv1alpha2.DefaultCalicoVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "calico")}, - "calico-flexvol": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "calico", Repo: "pod2daemon-flexvol", Tag: kubekeyv1alpha2.DefaultCalicoVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "calico")}, - "calico-typha": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "calico", Repo: "typha", Tag: kubekeyv1alpha2.DefaultCalicoVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "calico") && len(runtime.GetHostsByRole(common.K8s)) > 50}, - "flannel": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "flannel", Repo: "flannel", Tag: kubekeyv1alpha2.DefaultFlannelVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "flannel")}, - "flannel-cni-plugin": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "flannel", Repo: "flannel-cni-plugin", Tag: kubekeyv1alpha2.DefaultFlannelCniPluginVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "flannel")}, - "cilium": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "cilium", Repo: "cilium", Tag: kubekeyv1alpha2.DefaultCiliumVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "cilium")}, - "cilium-operator-generic": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "cilium", Repo: "operator-generic", Tag: kubekeyv1alpha2.DefaultCiliumVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "cilium")}, - "hybridnet": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "hybridnetdev", Repo: "hybridnet", Tag: kubekeyv1alpha2.DefaulthybridnetVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "hybridnet")}, - "kubeovn": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "kubeovn", Repo: "kube-ovn", Tag: kubekeyv1alpha2.DefaultKubeovnVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "kubeovn")}, - "multus": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "multus-cni", Tag: kubekeyv1alpha2.DefalutMultusVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.Contains(kubeConf.Cluster.Network.Plugin, "multus")}, - // storage - "provisioner-localpv": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "openebs", Repo: "provisioner-localpv", Tag: "3.3.0", Group: kubekeyv1alpha2.Worker, Enable: false}, - "linux-utils": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "openebs", Repo: "linux-utils", Tag: "3.3.0", Group: kubekeyv1alpha2.Worker, Enable: false}, - // load balancer - "haproxy": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "library", Repo: "haproxy", Tag: "2.3", Group: kubekeyv1alpha2.Worker, Enable: kubeConf.Cluster.ControlPlaneEndpoint.IsInternalLBEnabled()}, - "kubevip": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "plndr", Repo: "kube-vip", Tag: "v0.5.0", Group: kubekeyv1alpha2.Master, Enable: kubeConf.Cluster.ControlPlaneEndpoint.IsInternalLBEnabledVip()}, - // kata-deploy - "kata-deploy": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "kata-deploy", Tag: "stable", Group: kubekeyv1alpha2.Worker, Enable: kubeConf.Cluster.Kubernetes.EnableKataDeploy()}, - // node-feature-discovery - "node-feature-discovery": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "node-feature-discovery", Tag: "v0.10.0", Group: kubekeyv1alpha2.K8s, Enable: kubeConf.Cluster.Kubernetes.EnableNodeFeatureDiscovery()}, - } - - image = ImageList[name] - if kubeConf.Cluster.Registry.NamespaceOverride != "" { - image.NamespaceOverride = kubeConf.Cluster.Registry.NamespaceOverride - } - return image -} - -type SaveImages struct { - common.ArtifactAction -} - -func (s *SaveImages) Execute(runtime connector.Runtime) error { - auths := registry.DockerRegistryAuthEntries(s.Manifest.Spec.ManifestRegistry.Auths) - - dirName := filepath.Join(runtime.GetWorkDir(), common.Artifact, "images") - if err := coreutil.Mkdir(dirName); err != nil { - return errors.Wrapf(errors.WithStack(err), "mkdir %s failed", dirName) - } - for _, image := range s.Manifest.Spec.Images { - if err := validateImageName(image); err != nil { - return err - } - imageFullName := strings.Split(image, "/") - repo := imageFullName[0] - auth := new(registry.DockerRegistryEntry) - if v, ok := auths[repo]; ok { - auth = v - } - - srcName := fmt.Sprintf("docker://%s", image) - for _, platform := range s.Manifest.Spec.Arches { - arch, variant := ParseArchVariant(platform) - // placeholder - if variant != "" { - variant = "-" + variant - } - // Ex: - // oci:./kubekey/artifact/images:kubesphere:kube-apiserver:v1.21.5-amd64 - // oci:./kubekey/artifact/images:kubesphere:kube-apiserver:v1.21.5-arm-v7 - destName := fmt.Sprintf("oci:%s:%s:%s-%s%s", dirName, imageFullName[1], suffixImageName(imageFullName[2:]), arch, variant) - logger.Log.Infof("Source: %s", srcName) - logger.Log.Infof("Destination: %s", destName) - - o := &CopyImageOptions{ - srcImage: &srcImageOptions{ - imageName: srcName, - dockerImage: dockerImageOptions{ - arch: arch, - variant: variant, - os: "linux", - username: auth.Username, - password: auth.Password, - SkipTLSVerify: auth.SkipTLSVerify, - dockerCertPath: auth.CertsPath, - }, - }, - destImage: &destImageOptions{ - imageName: destName, - dockerImage: dockerImageOptions{ - arch: arch, - variant: variant, - os: "linux", - }, - }, - } - - if err := o.Copy(); err != nil { - return err - } - } - } - return nil -} - -type CopyImagesToRegistry struct { - common.KubeAction - ImagesPath string -} - -func (c *CopyImagesToRegistry) Execute(runtime connector.Runtime) error { - var imagesPath string - if c.ImagesPath != "" { - imagesPath = c.ImagesPath - } else { - imagesPath = filepath.Join(runtime.GetWorkDir(), "images") - } - - indexFile, err := os.ReadFile(filepath.Join(imagesPath, "index.json")) - if err != nil { - return errors.Errorf("read index.json failed: %s", err) - } - - index := NewIndex() - if err := json.Unmarshal(indexFile, index); err != nil { - return errors.Wrap(errors.WithStack(err), "unmarshal index.json failed: %s") - } - - auths := registry.DockerRegistryAuthEntries(c.KubeConf.Cluster.Registry.Auths) - - manifestList := make(map[string][]manifesttypes.ManifestEntry) - for _, m := range index.Manifests { - ref := m.Annotations.RefName - - // Ex: - // calico:cni:v3.20.0-amd64 - nameArr := strings.Split(ref, ":") - if len(nameArr) != 3 { - return errors.Errorf("invalid ref name: %s", ref) - } - - image := Image{ - RepoAddr: c.KubeConf.Cluster.Registry.PrivateRegistry, - Namespace: nameArr[0], - NamespaceOverride: c.KubeConf.Cluster.Registry.NamespaceOverride, - Repo: nameArr[1], - Tag: nameArr[2], - } - - uniqueImage, p := ParseImageWithArchTag(image.ImageName()) - entry := manifesttypes.ManifestEntry{ - Image: image.ImageName(), - Platform: p, - } - - skip := false - if v, ok := manifestList[uniqueImage]; ok { - // skip if the image already copied - for _, old := range v { - if reflect.DeepEqual(old, entry) { - skip = true - break - } - } - - if !skip { - v = append(v, entry) - manifestList[uniqueImage] = v - } - } else { - entryArr := make([]manifesttypes.ManifestEntry, 0) - manifestList[uniqueImage] = append(entryArr, entry) - } - - auth := new(registry.DockerRegistryEntry) - if config, ok := auths[c.KubeConf.Cluster.Registry.PrivateRegistry]; ok { - auth = config - } - - srcName := fmt.Sprintf("oci:%s:%s", imagesPath, ref) - destName := fmt.Sprintf("docker://%s", image.ImageName()) - logger.Log.Infof("Source: %s", srcName) - logger.Log.Infof("Destination: %s", destName) - - o := &CopyImageOptions{ - srcImage: &srcImageOptions{ - imageName: srcName, - dockerImage: dockerImageOptions{ - arch: p.Architecture, - variant: p.Variant, - os: "linux", - }, - }, - destImage: &destImageOptions{ - imageName: destName, - dockerImage: dockerImageOptions{ - arch: p.Architecture, - variant: p.Variant, - os: "linux", - username: auth.Username, - password: auth.Password, - SkipTLSVerify: auth.SkipTLSVerify, - dockerCertPath: auth.CertsPath, - }, - }, - } - - retry, maxRetry := 0, 5 - for ; retry < maxRetry; retry++ { - if err := o.Copy(); err == nil { - break - } - } - if retry >= maxRetry { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("copy image %s to %s failed, retry %d", srcName, destName, maxRetry)) - } - } - - c.ModuleCache.Set("manifestList", manifestList) - - return nil -} - -type PushManifest struct { - common.KubeAction -} - -func (p *PushManifest) Execute(_ connector.Runtime) error { - // make a multi-arch image - // push a manifest list to the private registry. - - v, ok := p.ModuleCache.Get("manifestList") - if !ok { - return errors.New("get manifest list failed by module cache") - } - list := v.(map[string][]manifesttypes.ManifestEntry) - - auths := registry.DockerRegistryAuthEntries(p.KubeConf.Cluster.Registry.Auths) - auth := new(registry.DockerRegistryEntry) - if _, ok := auths[p.KubeConf.Cluster.Registry.PrivateRegistry]; ok { - auth = auths[p.KubeConf.Cluster.Registry.PrivateRegistry] - } - - for imageName, platforms := range list { - manifestSpec := NewManifestSpec(imageName, platforms) - logger.Log.Debug(manifestSpec) - - logger.Log.Infof("Push multi-arch manifest list: %s", imageName) - // todo: the function can't support specify a certs dir - digest, length, err := manifestregistry.PushManifestList(auth.Username, auth.Password, manifestSpec, - false, true, auth.PlainHTTP, "") - if err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("push image %s multi-arch manifest failed", imageName)) - } - logger.Log.Infof("Digest: %s Length: %d", digest, length) - } - - return nil -} diff --git a/cmd/kk/pkg/images/utils.go b/cmd/kk/pkg/images/utils.go deleted file mode 100644 index 7e7a3ea66..000000000 --- a/cmd/kk/pkg/images/utils.go +++ /dev/null @@ -1,190 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package images - -import ( - "fmt" - "strings" - - "github.com/containerd/containerd/platforms" - "github.com/containers/image/v5/types" - manifesttypes "github.com/estesp/manifest-tool/v2/pkg/types" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" -) - -var defaultUserAgent = "kubekey" - -type dockerImageOptions struct { - arch string - os string - variant string - username string - password string - dockerCertPath string - SkipTLSVerify bool -} - -func (d *dockerImageOptions) systemContext() *types.SystemContext { - ctx := &types.SystemContext{ - ArchitectureChoice: d.arch, - OSChoice: d.os, - VariantChoice: d.variant, - DockerRegistryUserAgent: defaultUserAgent, - DockerInsecureSkipTLSVerify: types.NewOptionalBool(d.SkipTLSVerify), - } - return ctx -} - -type srcImageOptions struct { - dockerImage dockerImageOptions - imageName string - sharedBlobDir string -} - -func (s *srcImageOptions) systemContext() *types.SystemContext { - ctx := s.dockerImage.systemContext() - ctx.DockerCertPath = s.dockerImage.dockerCertPath - ctx.OCISharedBlobDirPath = s.sharedBlobDir - ctx.DockerAuthConfig = &types.DockerAuthConfig{ - Username: s.dockerImage.username, - Password: s.dockerImage.password, - } - - return ctx -} - -type destImageOptions struct { - dockerImage dockerImageOptions - imageName string -} - -func (d *destImageOptions) systemContext() *types.SystemContext { - ctx := d.dockerImage.systemContext() - ctx.DockerCertPath = d.dockerImage.dockerCertPath - ctx.DockerAuthConfig = &types.DockerAuthConfig{ - Username: d.dockerImage.username, - Password: d.dockerImage.password, - } - - return ctx -} - -// ParseArchVariant -// Ex: -// amd64 returns amd64, "" -// arm/v8 returns arm, v8 -func ParseArchVariant(platform string) (string, string) { - osArchArr := strings.Split(platform, "/") - - variant := "" - arch := osArchArr[0] - if len(osArchArr) > 1 { - variant = osArchArr[1] - } - return arch, variant -} - -func ParseImageWithArchTag(ref string) (string, ocispec.Platform) { - n := strings.LastIndex(ref, "-") - if n < 0 { - logger.Log.Fatalf("get arch or variant index failed: %s", ref) - } - archOrVariant := ref[n+1:] - - // try to parse the arch-only case - specifier := fmt.Sprintf("linux/%s", archOrVariant) - if p, err := platforms.Parse(specifier); err == nil && isKnownArch(p.Architecture) { - return ref[:n], p - } - - archStr := ref[:n] - a := strings.LastIndex(archStr, "-") - if a < 0 { - logger.Log.Fatalf("get arch index failed: %s", ref) - } - arch := archStr[a+1:] - - // parse the case where both arch and variant exist - specifier = fmt.Sprintf("linux/%s/%s", arch, archOrVariant) - p, err := platforms.Parse(specifier) - if err != nil { - logger.Log.Fatalf("parse image %s failed: %s", ref, err.Error()) - } - - return ref[:a], p -} - -func isKnownArch(arch string) bool { - switch arch { - case "386", "amd64", "amd64p32", "arm", "armbe", "arm64", "arm64be", "ppc64", "ppc64le", "loong64", "mips", "mipsle", "mips64", "mips64le", "mips64p32", "mips64p32le", "ppc", "riscv", "riscv64", "s390", "s390x", "sparc", "sparc64", "wasm": - return true - } - return false -} - -// ParseImageTag -// Get a repos name and returns the right reposName + tag -// The tag can be confusing because of a port in a repository name. -// -// Ex: localhost.localdomain:5000/samalba/hipache:latest -func ParseImageTag(repos string) (string, string) { - n := strings.LastIndex(repos, ":") - if n < 0 { - return repos, "" - } - if tag := repos[n+1:]; !strings.Contains(tag, "/") { - return repos[:n], tag - } - return repos, "" -} - -func NewManifestSpec(image string, entries []manifesttypes.ManifestEntry) manifesttypes.YAMLInput { - var srcImages []manifesttypes.ManifestEntry - - for _, e := range entries { - srcImages = append(srcImages, manifesttypes.ManifestEntry{ - Image: e.Image, - Platform: e.Platform, - }) - } - - return manifesttypes.YAMLInput{ - Image: image, - Manifests: srcImages, - } -} - -func validateImageName(imageFullName string) error { - image := strings.Split(imageFullName, "/") - if len(image) < 3 { - return errors.Errorf("image %s is invalid, image PATH need contain at least two slash-separated", imageFullName) - } - if len(strings.Split(image[len(image)-1], ":")) != 2 { - return errors.Errorf(`image %s is invalid, image PATH need contain ":"`, imageFullName) - } - return nil -} - -func suffixImageName(imageFullName []string) string { - if len(imageFullName) >= 2 { - return strings.Join(imageFullName, "/") - } - return imageFullName[0] -} diff --git a/cmd/kk/pkg/images/utils_test.go b/cmd/kk/pkg/images/utils_test.go deleted file mode 100644 index d393e8fff..000000000 --- a/cmd/kk/pkg/images/utils_test.go +++ /dev/null @@ -1,172 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package images - -import ( - "reflect" - "testing" - - v1 "github.com/opencontainers/image-spec/specs-go/v1" -) - -func TestParseArchVariant(t *testing.T) { - type args struct { - platform string - } - tests := []struct { - name string - args args - want string - want1 string - }{ - { - name: "test1", - args: args{ - platform: "amd64", - }, - want: "amd64", - want1: "", - }, - { - name: "test2", - args: args{ - platform: "arm/v8", - }, - want: "arm", - want1: "v8", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, got1 := ParseArchVariant(tt.args.platform) - if got != tt.want { - t.Errorf("ParseArchVariant() got = %v, want %v", got, tt.want) - } - if got1 != tt.want1 { - t.Errorf("ParseArchVariant() got1 = %v, want %v", got1, tt.want1) - } - }) - } -} - -func TestParseRepositoryTag(t *testing.T) { - type args struct { - repos string - } - tests := []struct { - name string - args args - want string - want1 string - }{ - { - name: "test1", - args: args{ - repos: "k8s.gcr.io/kube-apiserver-amd64:v1.16.3", - }, - want: "k8s.gcr.io/kube-apiserver-amd64", - want1: "v1.16.3", - }, - { - name: "test2", - args: args{ - repos: "docker.io/kubesphere/kube-apiserver:v1.16.3", - }, - want: "docker.io/kubesphere/kube-apiserver", - want1: "v1.16.3", - }, - { - name: "test3", - args: args{ - repos: "kube-apiserver:v1.16.3", - }, - want: "kube-apiserver", - want1: "v1.16.3", - }, - { - name: "test4", - args: args{ - repos: "calico:cni:v3.20.0-armv7", - }, - want: "calico:cni", - want1: "v3.20.0-armv7", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, got1 := ParseImageTag(tt.args.repos) - if got != tt.want { - t.Errorf("ParseImageTag() got = %v, want %v", got, tt.want) - } - if got1 != tt.want1 { - t.Errorf("ParseImageTag() got1 = %v, want %v", got1, tt.want1) - } - }) - } -} - -func TestParseImageWithArchTag(t *testing.T) { - tests := []struct { - name string - ref string - want1 string - want2 v1.Platform - }{ - { - name: "t1", - ref: "kube-apiserver:v1.21.5-amd64", - want1: "kube-apiserver:v1.21.5", - want2: v1.Platform{ - OS: "linux", - Architecture: "amd64", - Variant: "", - }, - }, - { - name: "t1", - ref: "kube-apiserver:v1.21.5-amd64", - want1: "kube-apiserver:v1.21.5", - want2: v1.Platform{ - OS: "linux", - Architecture: "amd64", - }, - }, - { - name: "t2", - ref: "kube-apiserver:v1.21.5-arm-v7", - want1: "kube-apiserver:v1.21.5", - want2: v1.Platform{ - OS: "linux", - Architecture: "arm", - Variant: "v7", - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, got1 := ParseImageWithArchTag(tt.ref) - - if got != tt.want1 { - t.Errorf("ParseImageTag() got = %v, want %v", got, tt.want1) - } - - if !reflect.DeepEqual(got1, tt.want2) { - t.Errorf("ParseImageWithArchTag() = %v, want %v", got, tt.want2) - } - }) - } -} diff --git a/cmd/kk/pkg/k3s/k3s_status.go b/cmd/kk/pkg/k3s/k3s_status.go deleted file mode 100644 index f93810ddb..000000000 --- a/cmd/kk/pkg/k3s/k3s_status.go +++ /dev/null @@ -1,124 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package k3s - -import ( - "fmt" - "os" - "path/filepath" - "regexp" - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type K3sStatus struct { - Version string - ClusterInfo string - NodeToken string - KubeConfig string - NodesInfo map[string]string -} - -func NewK3sStatus() *K3sStatus { - return &K3sStatus{NodesInfo: make(map[string]string)} -} - -func (k *K3sStatus) SearchVersion(runtime connector.Runtime) error { - cmd := "k3s --version | grep 'k3s' | awk '{print $3}'" - if output, err := runtime.GetRunner().Cmd(cmd, true); err != nil { - return errors.Wrap(errors.WithStack(err), "search current version failed") - } else { - k.Version = output - } - return nil -} - -func (k *K3sStatus) SearchKubeConfig(runtime connector.Runtime) error { - kubeCfgCmd := "cat /etc/rancher/k3s/k3s.yaml" - if kubeConfigStr, err := runtime.GetRunner().SudoCmd(kubeCfgCmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "search cluster kubeconfig failed") - } else { - k.KubeConfig = kubeConfigStr - } - return nil -} - -func (k *K3sStatus) SearchNodeToken(runtime connector.Runtime) error { - nodeTokenBase64Cmd := "cat /var/lib/rancher/k3s/server/node-token" - output, err := runtime.GetRunner().SudoCmd(nodeTokenBase64Cmd, true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get cluster node token failed") - } - k.NodeToken = output - return nil -} - -func (k *K3sStatus) SearchInfo(runtime connector.Runtime) error { - output, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl --no-headers=true get nodes -o custom-columns=:metadata.name,:status.nodeInfo.kubeletVersion,:status.addresses", - true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get k3s cluster info failed") - } - k.ClusterInfo = output - return nil -} - -func (k *K3sStatus) SearchNodesInfo(_ connector.Runtime) error { - ipv4Regexp, err := regexp.Compile(common.IPv4Regexp) - if err != nil { - return err - } - ipv6Regexp, err := regexp.Compile(common.IPv6Regexp) - if err != nil { - return err - } - tmp := strings.Split(k.ClusterInfo, "\r\n") - if len(tmp) >= 1 { - for i := 0; i < len(tmp); i++ { - if ipv4 := ipv4Regexp.FindStringSubmatch(tmp[i]); len(ipv4) != 0 { - k.NodesInfo[ipv4[0]] = ipv4[0] - } - if ipv6 := ipv6Regexp.FindStringSubmatch(tmp[i]); len(ipv6) != 0 { - k.NodesInfo[ipv6[0]] = ipv6[0] - } - if len(strings.Fields(tmp[i])) > 3 { - k.NodesInfo[strings.Fields(tmp[i])[0]] = strings.Fields(tmp[i])[1] - } else { - k.NodesInfo[strings.Fields(tmp[i])[0]] = "" - } - } - } - return nil -} - -func (k *K3sStatus) LoadKubeConfig(runtime connector.Runtime, kubeConf *common.KubeConf) error { - kubeConfigPath := filepath.Join(runtime.GetWorkDir(), fmt.Sprintf("config-%s", runtime.GetObjName())) - - oldServer := "server: https://127.0.0.1:6443" - newServer := fmt.Sprintf("server: https://%s:%d", kubeConf.Cluster.ControlPlaneEndpoint.Address, kubeConf.Cluster.ControlPlaneEndpoint.Port) - newKubeConfigStr := strings.Replace(k.KubeConfig, oldServer, newServer, -1) - - if err := os.WriteFile(kubeConfigPath, []byte(newKubeConfigStr), 0644); err != nil { - return err - } - return nil -} diff --git a/cmd/kk/pkg/k3s/module.go b/cmd/kk/pkg/k3s/module.go deleted file mode 100644 index 9520d88e8..000000000 --- a/cmd/kk/pkg/k3s/module.go +++ /dev/null @@ -1,343 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package k3s - -import ( - "path/filepath" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/k3s/templates" -) - -type StatusModule struct { - common.KubeModule -} - -func (s *StatusModule) Init() { - s.Name = "StatusModule" - s.Desc = "Get cluster status" - - cluster := NewK3sStatus() - s.PipelineCache.GetOrSet(common.ClusterStatus, cluster) - - clusterStatus := &task.RemoteTask{ - Name: "GetClusterStatus", - Desc: "Get k3s cluster status", - Hosts: s.Runtime.GetHostsByRole(common.Master), - Action: new(GetClusterStatus), - Parallel: false, - } - - s.Tasks = []task.Interface{ - clusterStatus, - } -} - -type InstallKubeBinariesModule struct { - common.KubeModule -} - -func (i *InstallKubeBinariesModule) Init() { - i.Name = "InstallKubeBinariesModule" - i.Desc = "Install k3s cluster" - - syncBinary := &task.RemoteTask{ - Name: "SyncKubeBinary", - Desc: "Synchronize k3s binaries", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: new(SyncKubeBinary), - Parallel: true, - Retry: 2, - } - - killAllScript := &task.RemoteTask{ - Name: "GenerateK3sKillAllScript", - Desc: "Generate k3s killall.sh script", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: &action.Template{ - Template: templates.K3sKillallScript, - Dst: filepath.Join("/usr/local/bin", templates.K3sKillallScript.Name()), - }, - Parallel: true, - Retry: 2, - } - - uninstallScript := &task.RemoteTask{ - Name: "GenerateK3sUninstallScript", - Desc: "Generate k3s uninstall script", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: &action.Template{ - Template: templates.K3sUninstallScript, - Dst: filepath.Join("/usr/local/bin", templates.K3sUninstallScript.Name()), - }, - Parallel: true, - Retry: 2, - } - - chmod := &task.RemoteTask{ - Name: "ChmodScript", - Desc: "Chmod +x k3s script ", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: new(ChmodScript), - Parallel: true, - Retry: 2, - } - - i.Tasks = []task.Interface{ - syncBinary, - killAllScript, - uninstallScript, - chmod, - } -} - -type InitClusterModule struct { - common.KubeModule -} - -func (i *InitClusterModule) Init() { - i.Name = "K3sInitClusterModule" - i.Desc = "Init k3s cluster" - - k3sService := &task.RemoteTask{ - Name: "GenerateK3sService", - Desc: "Generate k3s Service", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: new(GenerateK3sService), - Parallel: true, - } - - k3sEnv := &task.RemoteTask{ - Name: "GenerateK3sServiceEnv", - Desc: "Generate k3s service env", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: new(GenerateK3sServiceEnv), - Parallel: true, - } - - k3sRegistryConfig := &task.RemoteTask{ - Name: "GenerateK3sRegistryConfig", - Desc: "Generate k3s registry config", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - &UsePrivateRegstry{Not: false}, - }, - Action: new(GenerateK3sRegistryConfig), - Parallel: true, - } - - enableK3s := &task.RemoteTask{ - Name: "EnableK3sService", - Desc: "Enable k3s service", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: new(EnableK3sService), - Parallel: true, - } - - copyKubeConfig := &task.RemoteTask{ - Name: "CopyKubeConfig", - Desc: "Copy k3s.yaml to ~/.kube/config", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: new(CopyK3sKubeConfig), - Parallel: true, - } - - addMasterTaint := &task.RemoteTask{ - Name: "AddMasterTaint", - Desc: "Add master taint", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - &common.IsWorker{Not: true}, - }, - Action: new(AddMasterTaint), - Parallel: true, - Retry: 5, - } - - i.Tasks = []task.Interface{ - k3sService, - k3sEnv, - k3sRegistryConfig, - enableK3s, - copyKubeConfig, - addMasterTaint, - } -} - -type JoinNodesModule struct { - common.KubeModule -} - -func (j *JoinNodesModule) Init() { - j.Name = "K3sJoinNodesModule" - j.Desc = "Join k3s nodes" - - k3sService := &task.RemoteTask{ - Name: "GenerateK3sService", - Desc: "Generate k3s Service", - Hosts: j.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - }, - Action: new(GenerateK3sService), - Parallel: true, - } - - k3sEnv := &task.RemoteTask{ - Name: "GenerateK3sServiceEnv", - Desc: "Generate k3s service env", - Hosts: j.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - }, - Action: new(GenerateK3sServiceEnv), - Parallel: true, - } - - k3sRegistryConfig := &task.RemoteTask{ - Name: "GenerateK3sRegistryConfig", - Desc: "Generate k3s registry config", - Hosts: j.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - &UsePrivateRegstry{Not: false}, - }, - Action: new(GenerateK3sRegistryConfig), - Parallel: true, - } - - enableK3s := &task.RemoteTask{ - Name: "EnableK3sService", - Desc: "Enable k3s service", - Hosts: j.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - }, - Action: new(EnableK3sService), - Parallel: true, - } - - copyKubeConfigForMaster := &task.RemoteTask{ - Name: "CopyKubeConfig", - Desc: "Copy k3s.yaml to ~/.kube/config", - Hosts: j.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - }, - Action: new(CopyK3sKubeConfig), - Parallel: true, - } - - addMasterTaint := &task.RemoteTask{ - Name: "AddMasterTaint", - Desc: "Add master taint", - Hosts: j.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - &common.IsWorker{Not: true}, - }, - Action: new(AddMasterTaint), - Parallel: true, - Retry: 5, - } - - addWorkerLabelToNode := &task.RemoteTask{ - Name: "addWorkerLabelToNode", - Desc: "Add worker label to all nodes", - Hosts: j.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(AddWorkerLabel), - Retry: 3, - } - - j.Tasks = []task.Interface{ - k3sService, - k3sEnv, - k3sRegistryConfig, - enableK3s, - copyKubeConfigForMaster, - addMasterTaint, - addWorkerLabelToNode, - } -} - -type DeleteClusterModule struct { - common.KubeModule -} - -func (d *DeleteClusterModule) Init() { - d.Name = "DeleteClusterModule" - d.Desc = "Delete k3s cluster" - - execScript := &task.RemoteTask{ - Name: "ExecUninstallScript", - Desc: "Exec k3s uninstall script", - Hosts: d.Runtime.GetHostsByRole(common.K8s), - Action: new(ExecUninstallScript), - Parallel: true, - } - - d.Tasks = []task.Interface{ - execScript, - } -} - -type SaveKubeConfigModule struct { - common.KubeModule -} - -func (s *SaveKubeConfigModule) Init() { - s.Name = "SaveKubeConfigModule" - s.Desc = "Save kube config file as a configmap" - - save := &task.LocalTask{ - Name: "SaveKubeConfig", - Desc: "Save kube config as a configmap", - Action: new(SaveKubeConfig), - } - - s.Tasks = []task.Interface{ - save, - } -} diff --git a/cmd/kk/pkg/k3s/prepares.go b/cmd/kk/pkg/k3s/prepares.go deleted file mode 100644 index 0ff3bd9d9..000000000 --- a/cmd/kk/pkg/k3s/prepares.go +++ /dev/null @@ -1,71 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package k3s - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type NodeInCluster struct { - common.KubePrepare - Not bool -} - -func (n *NodeInCluster) PreCheck(runtime connector.Runtime) (bool, error) { - host := runtime.RemoteHost() - if v, ok := n.PipelineCache.Get(common.ClusterStatus); ok { - cluster := v.(*K3sStatus) - var versionOk bool - if res, ok := cluster.NodesInfo[host.GetName()]; ok && res != "" { - versionOk = true - } - _, ipOk := cluster.NodesInfo[host.GetInternalAddress()] - if n.Not { - return !(versionOk || ipOk), nil - } - return versionOk || ipOk, nil - } - return false, errors.New("get k3s cluster status by pipeline cache failed") -} - -type ClusterIsExist struct { - common.KubePrepare - Not bool -} - -func (c *ClusterIsExist) PreCheck(_ connector.Runtime) (bool, error) { - if exist, ok := c.PipelineCache.GetMustBool(common.ClusterExist); ok { - if c.Not { - return !exist, nil - } - return exist, nil - } else { - return false, errors.New("get k3s cluster status by pipeline cache failed") - } -} - -type UsePrivateRegstry struct { - common.KubePrepare - Not bool -} - -func (c *UsePrivateRegstry) PreCheck(_ connector.Runtime) (bool, error) { - return c.KubeConf.Cluster.Registry.PrivateRegistry != "", nil -} diff --git a/cmd/kk/pkg/k3s/tasks.go b/cmd/kk/pkg/k3s/tasks.go deleted file mode 100644 index 9b68c81d8..000000000 --- a/cmd/kk/pkg/k3s/tasks.go +++ /dev/null @@ -1,531 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package k3s - -import ( - "context" - "encoding/base64" - "fmt" - "path/filepath" - "strings" - - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - versionutil "k8s.io/apimachinery/pkg/util/version" - kube "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/k3s/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/registry" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" -) - -type GetClusterStatus struct { - common.KubeAction -} - -func (g *GetClusterStatus) Execute(runtime connector.Runtime) error { - exist, err := runtime.GetRunner().FileExist("/etc/systemd/system/k3s.service") - if err != nil { - return err - } - - if !exist { - g.PipelineCache.Set(common.ClusterExist, false) - return nil - } else { - g.PipelineCache.Set(common.ClusterExist, true) - - if v, ok := g.PipelineCache.Get(common.ClusterStatus); ok { - cluster := v.(*K3sStatus) - if err := cluster.SearchVersion(runtime); err != nil { - return err - } - if err := cluster.SearchKubeConfig(runtime); err != nil { - return err - } - if err := cluster.LoadKubeConfig(runtime, g.KubeConf); err != nil { - return err - } - if err := cluster.SearchNodeToken(runtime); err != nil { - return err - } - if err := cluster.SearchInfo(runtime); err != nil { - return err - } - if err := cluster.SearchNodesInfo(runtime); err != nil { - return err - } - g.PipelineCache.Set(common.ClusterStatus, cluster) - } else { - return errors.New("get k3s cluster status by pipeline cache failed") - } - } - return nil -} - -type SyncKubeBinary struct { - common.KubeAction -} - -func (s *SyncKubeBinary) Execute(runtime connector.Runtime) error { - binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) - if !ok { - return errors.New("get KubeBinary by pipeline cache failed") - } - binariesMap := binariesMapObj.(map[string]*files.KubeBinary) - - if err := SyncKubeBinaries(s, runtime, binariesMap); err != nil { - return err - } - return nil -} - -// SyncKubeBinaries is used to sync kubernetes' binaries to each node. -func SyncKubeBinaries(s *SyncKubeBinary, runtime connector.Runtime, binariesMap map[string]*files.KubeBinary) error { - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - binaryList := []string{"k3s", "helm", "kubecni"} - if s.KubeConf.Cluster.Network.Plugin == "calico" { - binaryList = append(binaryList, "calicoctl") - } - for _, name := range binaryList { - binary, ok := binariesMap[name] - if !ok { - return fmt.Errorf("get kube binary %s info failed: no such key", name) - } - - fileName := binary.FileName - switch name { - case "kubecni": - dst := filepath.Join(common.TmpDir, fileName) - if err := runtime.GetRunner().Scp(binary.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync kube binaries failed")) - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("tar -zxf %s -C /opt/cni/bin", dst), false); err != nil { - return err - } - default: - dst := filepath.Join(common.BinDir, fileName) - if err := runtime.GetRunner().SudoScp(binary.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync kube binaries failed")) - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chmod +x %s", dst), false); err != nil { - return err - } - } - } - - binaries := []string{"kubectl", "crictl", "ctr"} - var createLinkCMDs []string - for _, binary := range binaries { - createLinkCMDs = append(createLinkCMDs, fmt.Sprintf("ln -snf /usr/local/bin/k3s /usr/local/bin/%s", binary)) - } - if _, err := runtime.GetRunner().SudoCmd(strings.Join(createLinkCMDs, " && "), false); err != nil { - return errors.Wrap(errors.WithStack(err), "create ctl tool link failed") - } - - return nil -} - -type ChmodScript struct { - common.KubeAction -} - -func (c *ChmodScript) Execute(runtime connector.Runtime) error { - killAllScript := filepath.Join("/usr/local/bin", templates.K3sKillallScript.Name()) - uninstallScript := filepath.Join("/usr/local/bin", templates.K3sUninstallScript.Name()) - - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chmod +x %s", killAllScript), - false); err != nil { - return err - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chmod +x %s", uninstallScript), - false); err != nil { - return err - } - return nil -} - -type GenerateK3sService struct { - common.KubeAction -} - -func (g *GenerateK3sService) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - - var server string - if !host.IsRole(common.Master) { - server = fmt.Sprintf("https://%s:%d", g.KubeConf.Cluster.ControlPlaneEndpoint.Domain, g.KubeConf.Cluster.ControlPlaneEndpoint.Port) - } - - defaultKubeletArs := map[string]string{ - "cni-conf-dir": "/etc/cni/net.d", - "cni-bin-dir": "/opt/cni/bin", - "kube-reserved": "cpu=200m,memory=250Mi,ephemeral-storage=1Gi", - "system-reserved": "cpu=200m,memory=250Mi,ephemeral-storage=1Gi", - "eviction-hard": "memory.available<5%,nodefs.available<10%", - } - defaultKubeProxyArgs := map[string]string{ - "proxy-mode": "ipvs", - } - - kubeApiserverArgs, _ := util.GetArgs(map[string]string{}, g.KubeConf.Cluster.Kubernetes.ApiServerArgs) - kubeControllerManager, _ := util.GetArgs(map[string]string{ - "pod-eviction-timeout": "3m0s", - "terminated-pod-gc-threshold": "5", - }, g.KubeConf.Cluster.Kubernetes.ControllerManagerArgs) - kubeSchedulerArgs, _ := util.GetArgs(map[string]string{}, g.KubeConf.Cluster.Kubernetes.SchedulerArgs) - kubeletArgs, _ := util.GetArgs(defaultKubeletArs, g.KubeConf.Cluster.Kubernetes.KubeletArgs) - kubeProxyArgs, _ := util.GetArgs(defaultKubeProxyArgs, g.KubeConf.Cluster.Kubernetes.KubeProxyArgs) - - templateAction := action.Template{ - Template: templates.K3sService, - Dst: filepath.Join("/etc/systemd/system/", templates.K3sService.Name()), - Data: util.Data{ - "Server": server, - "IsMaster": host.IsRole(common.Master), - "IsDockerRuntime": g.KubeConf.Cluster.Kubernetes.ContainerManager == common.Docker, - "ContainerRuntimeEndpoint": g.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint, - "NodeIP": host.GetInternalAddress(), - "HostName": host.GetName(), - "PodSubnet": g.KubeConf.Cluster.Network.KubePodsCIDR, - "ServiceSubnet": g.KubeConf.Cluster.Network.KubeServiceCIDR, - "ClusterDns": g.KubeConf.Cluster.CorednsClusterIP(), - "CertSANs": g.KubeConf.Cluster.GenerateCertSANs(), - "PauseImage": images.GetImage(runtime, g.KubeConf, "pause").ImageName(), - "ApiserverArgs": kubeApiserverArgs, - "ControllerManager": kubeControllerManager, - "SchedulerArgs": kubeSchedulerArgs, - "KubeletArgs": kubeletArgs, - "KubeProxyArgs": kubeProxyArgs, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type GenerateK3sServiceEnv struct { - common.KubeAction -} - -func (g *GenerateK3sServiceEnv) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - - clusterStatus, ok := g.PipelineCache.Get(common.ClusterStatus) - if !ok { - return errors.New("get cluster status by pipeline cache failed") - } - cluster := clusterStatus.(*K3sStatus) - - var externalEtcd kubekeyapiv1alpha2.ExternalEtcd - var endpointsList []string - var externalEtcdEndpoints, token string - - switch g.KubeConf.Cluster.Etcd.Type { - case kubekeyapiv1alpha2.External: - externalEtcd.Endpoints = g.KubeConf.Cluster.Etcd.External.Endpoints - - if len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 && len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 && len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 { - externalEtcd.CAFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.CAFile)) - externalEtcd.CertFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.CertFile)) - externalEtcd.KeyFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.KeyFile)) - } - default: - for _, node := range runtime.GetHostsByRole(common.ETCD) { - endpoint := fmt.Sprintf("https://%s:%s", node.GetInternalAddress(), kubekeyapiv1alpha2.DefaultEtcdPort) - endpointsList = append(endpointsList, endpoint) - } - externalEtcd.Endpoints = endpointsList - - externalEtcd.CAFile = "/etc/ssl/etcd/ssl/ca.pem" - externalEtcd.CertFile = fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s.pem", runtime.GetHostsByRole(common.Master)[0].GetName()) - externalEtcd.KeyFile = fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s-key.pem", runtime.GetHostsByRole(common.Master)[0].GetName()) - } - - externalEtcdEndpoints = strings.Join(externalEtcd.Endpoints, ",") - - v121 := versionutil.MustParseSemantic("v1.21.0") - atLeast := versionutil.MustParseSemantic(g.KubeConf.Cluster.Kubernetes.Version).AtLeast(v121) - if atLeast { - token = cluster.NodeToken - } else { - if !host.IsRole(common.Master) { - token = cluster.NodeToken - } - } - - templateAction := action.Template{ - Template: templates.K3sServiceEnv, - Dst: filepath.Join("/etc/systemd/system/", templates.K3sServiceEnv.Name()), - Data: util.Data{ - "DataStoreEndPoint": externalEtcdEndpoints, - "DataStoreCaFile": externalEtcd.CAFile, - "DataStoreCertFile": externalEtcd.CertFile, - "DataStoreKeyFile": externalEtcd.KeyFile, - "IsMaster": host.IsRole(common.Master), - "Token": token, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type EnableK3sService struct { - common.KubeAction -} - -func (e *EnableK3sService) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && systemctl enable --now k3s", - false); err != nil { - return errors.Wrap(errors.WithStack(err), "enable k3s failed") - } - return nil -} - -type CopyK3sKubeConfig struct { - common.KubeAction -} - -func (c *CopyK3sKubeConfig) Execute(runtime connector.Runtime) error { - createConfigDirCmd := "mkdir -p /root/.kube && mkdir -p $HOME/.kube" - getKubeConfigCmd := "cp -f /etc/rancher/k3s/k3s.yaml /root/.kube/config" - - cmd := strings.Join([]string{createConfigDirCmd, getKubeConfigCmd}, " && ") - if _, err := runtime.GetRunner().SudoCmd(cmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "copy k3s kube config failed") - } - - userMkdir := "mkdir -p $HOME/.kube" - if _, err := runtime.GetRunner().Cmd(userMkdir, false); err != nil { - return errors.Wrap(errors.WithStack(err), "user mkdir $HOME/.kube failed") - } - - userCopyKubeConfig := "cp -f /etc/rancher/k3s/k3s.yaml $HOME/.kube/config" - if _, err := runtime.GetRunner().SudoCmd(userCopyKubeConfig, false); err != nil { - return errors.Wrap(errors.WithStack(err), "user copy /etc/rancher/k3s/k3s.yaml to $HOME/.kube/config failed") - } - - userId, err := runtime.GetRunner().Cmd("echo $(id -u)", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get user id failed") - } - - userGroupId, err := runtime.GetRunner().Cmd("echo $(id -g)", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get user group id failed") - } - - chownKubeConfig := fmt.Sprintf("chown -R %s:%s $HOME/.kube", userId, userGroupId) - if _, err := runtime.GetRunner().SudoCmd(chownKubeConfig, false); err != nil { - return errors.Wrap(errors.WithStack(err), "chown user kube config failed") - } - return nil -} - -type AddMasterTaint struct { - common.KubeAction -} - -func (a *AddMasterTaint) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - - cmd := fmt.Sprintf( - "/usr/local/bin/kubectl taint nodes %s node-role.kubernetes.io/master=effect:NoSchedule --overwrite", - host.GetName()) - - if _, err := runtime.GetRunner().SudoCmd(cmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "add master NoSchedule taint failed") - } - return nil -} - -type AddWorkerLabel struct { - common.KubeAction -} - -func (a *AddWorkerLabel) Execute(runtime connector.Runtime) error { - for _, host := range runtime.GetAllHosts() { - if host.IsRole(common.Worker) { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "/usr/local/bin/kubectl label --overwrite node %s node-role.kubernetes.io/worker=", - host.GetName()), true); err != nil { - return errors.Wrap(errors.WithStack(err), "add worker label failed") - } - } - } - - return nil -} - -type ExecUninstallScript struct { - common.KubeAction -} - -func (e *ExecUninstallScript) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && /usr/local/bin/k3s-killall.sh", - true); err != nil { - return errors.Wrap(errors.WithStack(err), "add master NoSchedule taint failed") - } - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && /usr/local/bin/k3s-uninstall.sh", - true); err != nil { - return errors.Wrap(errors.WithStack(err), "add master NoSchedule taint failed") - } - return nil -} - -type SaveKubeConfig struct { - common.KubeAction -} - -func (s *SaveKubeConfig) Execute(_ connector.Runtime) error { - status, ok := s.PipelineCache.Get(common.ClusterStatus) - if !ok { - return errors.New("get kubernetes status failed by pipeline cache") - } - cluster := status.(*K3sStatus) - - oldServer := fmt.Sprintf("https://%s:%d", s.KubeConf.Cluster.ControlPlaneEndpoint.Domain, s.KubeConf.Cluster.ControlPlaneEndpoint.Port) - newServer := fmt.Sprintf("https://%s:%d", s.KubeConf.Cluster.ControlPlaneEndpoint.Address, s.KubeConf.Cluster.ControlPlaneEndpoint.Port) - newKubeConfigStr := strings.Replace(cluster.KubeConfig, oldServer, newServer, -1) - kubeConfigBase64 := base64.StdEncoding.EncodeToString([]byte(newKubeConfigStr)) - - config, err := clientcmd.NewClientConfigFromBytes([]byte(newKubeConfigStr)) - if err != nil { - return err - } - restConfig, err := config.ClientConfig() - if err != nil { - return err - } - clientsetForCluster, err := kube.NewForConfig(restConfig) - if err != nil { - return err - } - - namespace := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kubekey-system", - }, - } - if _, err := clientsetForCluster. - CoreV1(). - Namespaces(). - Create(context.TODO(), namespace, metav1.CreateOptions{}); err != nil { - return err - } - - cm := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-kubeconfig", s.KubeConf.ClusterName), - }, - Data: map[string]string{ - "kubeconfig": kubeConfigBase64, - }, - } - - if _, err := clientsetForCluster. - CoreV1(). - ConfigMaps("kubekey-system"). - Create(context.TODO(), cm, metav1.CreateOptions{}); err != nil { - return err - } - return nil -} - -type GenerateK3sRegistryConfig struct { - common.KubeAction -} - -func (g *GenerateK3sRegistryConfig) Execute(runtime connector.Runtime) error { - endpointPrefix := "https://" - dockerioMirror := registry.Mirror{} - registryConfigs := map[string]registry.RegistryConfig{} - - auths := registry.DockerRegistryAuthEntries(g.KubeConf.Cluster.Registry.Auths) - for k, v := range auths { - if k == g.KubeConf.Cluster.Registry.PrivateRegistry && v.PlainHTTP { - endpointPrefix = "http://" - } - } - - dockerioMirror.Endpoints = []string{fmt.Sprintf("%s%s", endpointPrefix, g.KubeConf.Cluster.Registry.PrivateRegistry)} - - if g.KubeConf.Cluster.Registry.NamespaceOverride != "" { - dockerioMirror.Rewrites = map[string]string{ - "^rancher/(.*)": fmt.Sprintf("%s/$1", g.KubeConf.Cluster.Registry.NamespaceOverride), - } - } - - for k, v := range auths { - registryConfigs[k] = registry.RegistryConfig{ - Auth: ®istry.AuthConfig{ - Username: v.Username, - Password: v.Password, - }, - TLS: ®istry.TLSConfig{ - CAFile: v.CAFile, - CertFile: v.CertFile, - KeyFile: v.KeyFile, - InsecureSkipVerify: v.SkipTLSVerify, - }, - } - } - - _, ok := registryConfigs[g.KubeConf.Cluster.Registry.PrivateRegistry] - - if !ok { - registryConfigs[g.KubeConf.Cluster.Registry.PrivateRegistry] = registry.RegistryConfig{TLS: ®istry.TLSConfig{InsecureSkipVerify: true}} - } - - k3sRegistries := registry.Registry{ - Mirrors: map[string]registry.Mirror{"docker.io": dockerioMirror}, - Configs: registryConfigs, - } - - templateAction := action.Template{ - Template: templates.K3sRegistryConfigTempl, - Dst: filepath.Join("/etc/rancher/k3s", templates.K3sRegistryConfigTempl.Name()), - Data: util.Data{ - "Registries": k3sRegistries, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} diff --git a/cmd/kk/pkg/k3s/templates/k3sRegistryConfig.go b/cmd/kk/pkg/k3s/templates/k3sRegistryConfig.go deleted file mode 100644 index cd5266d84..000000000 --- a/cmd/kk/pkg/k3s/templates/k3sRegistryConfig.go +++ /dev/null @@ -1,32 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" -) - -var ( - funcMap = template.FuncMap{"toYaml": utils.ToYAML, "indent": utils.Indent} - // k3sRegistryConfigTempl defines the template of k3s' registry. - K3sRegistryConfigTempl = template.Must(template.New("registries.yaml").Funcs(funcMap).Parse( - dedent.Dedent(`{{ toYaml .Registries }}`))) -) diff --git a/cmd/kk/pkg/k3s/templates/k3sService.go b/cmd/kk/pkg/k3s/templates/k3sService.go deleted file mode 100644 index b43a209c9..000000000 --- a/cmd/kk/pkg/k3s/templates/k3sService.go +++ /dev/null @@ -1,59 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -var ( - // K3sService defines the template of kubelet service for systemd. - K3sService = template.Must(template.New("k3s.service").Parse( - dedent.Dedent(`[Unit] -Description=Lightweight Kubernetes -Documentation=https://k3s.io -Wants=network-online.target -After=network-online.target - -[Install] -WantedBy=multi-user.target - -[Service] -Type=notify -EnvironmentFile=/etc/systemd/system/k3s.service.env -{{ if .IsMaster }} -Environment="K3S_ARGS={{ range .CertSANs }} --tls-san={{ . }}{{- end }} {{ range .ApiserverArgs }} --kube-apiserver-arg={{ . }}{{- end }} {{ range .ControllerManager }} --kube-controller-manager-arg={{ . }}{{- end }} {{ range .SchedulerArgs }} --kube-scheduler-arg={{ . }}{{- end }} --cluster-cidr={{ .PodSubnet }} --service-cidr={{ .ServiceSubnet }} --cluster-dns={{ .ClusterDns }} --flannel-backend=none --disable-network-policy --disable-cloud-controller --disable=servicelb,traefik,metrics-server,local-storage" -{{ end }} -Environment="K3S_EXTRA_ARGS={{ if .IsDockerRuntime }}--docker{{ end }} {{ if .ContainerRuntimeEndpoint }}--container-runtime-endpoint={{ .ContainerRuntimeEndpoint }}{{ end }} --node-name={{ .HostName }} --node-ip={{ .NodeIP }} --pause-image={{ .PauseImage }} {{ range .KubeletArgs }} --kubelet-arg={{ . }}{{- end }} {{ range .KubeProxyArgs }} --kube-proxy-arg={{ . }}{{- end }}" -Environment="K3S_ROLE={{ if .IsMaster }}server{{ else }}agent{{ end }}" -Environment="K3S_SERVER_ARGS={{ if .Server }}--server={{ .Server }}{{ end }}" -KillMode=process -Delegate=yes -LimitNOFILE=1048576 -LimitNPROC=infinity -LimitCORE=infinity -TasksMax=infinity -TimeoutStartSec=0 -Restart=always -RestartSec=5s -ExecStartPre=-/sbin/modprobe br_netfilter -ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/local/bin/k3s $K3S_ROLE $K3S_ARGS $K3S_EXTRA_ARGS $K3S_SERVER_ARGS - `))) -) diff --git a/cmd/kk/pkg/k3s/templates/k3sServiceEnv.go b/cmd/kk/pkg/k3s/templates/k3sServiceEnv.go deleted file mode 100644 index 49e92614d..000000000 --- a/cmd/kk/pkg/k3s/templates/k3sServiceEnv.go +++ /dev/null @@ -1,44 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -// K3sServiceEnv defines the template of kubelet's Env for the kubelet's systemd service. -var K3sServiceEnv = template.Must(template.New("k3s.service.env").Parse( - dedent.Dedent(`# Note: This dropin only works with k3s -{{ if .IsMaster }} -K3S_DATASTORE_ENDPOINT={{ .DataStoreEndPoint }} -{{- if .DataStoreCaFile }} -K3S_DATASTORE_CAFILE={{ .DataStoreCaFile }} -{{- end }} -{{- if .DataStoreCertFile }} -K3S_DATASTORE_CERTFILE={{ .DataStoreCertFile }} -{{- end }} -{{- if .DataStoreKeyFile }} -K3S_DATASTORE_KEYFILE={{ .DataStoreKeyFile }} -{{- end }} -K3S_KUBECONFIG_MODE=644 -{{ end }} -{{ if .Token }} -K3S_TOKEN={{ .Token }} -{{ end }} - - `))) diff --git a/cmd/kk/pkg/k3s/templates/killall_script.go b/cmd/kk/pkg/k3s/templates/killall_script.go deleted file mode 100644 index 12eb709a2..000000000 --- a/cmd/kk/pkg/k3s/templates/killall_script.go +++ /dev/null @@ -1,109 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -// K3sKillallScript defines the template of k3s-killall script. -var K3sKillallScript = template.Must(template.New("k3s-killall.sh").Parse( - dedent.Dedent(`#!/bin/sh -[ $(id -u) -eq 0 ] || exec sudo $0 $@ - -for bin in /var/lib/rancher/k3s/data/**/bin/; do - [ -d $bin ] && export PATH=$PATH:$bin:$bin/aux -done - -set -x - -for service in /etc/systemd/system/k3s*.service; do - [ -s $service ] && systemctl stop $(basename $service) -done - -for service in /etc/init.d/k3s*; do - [ -x $service ] && $service stop -done - -pschildren() { - ps -e -o ppid= -o pid= | \ - sed -e 's/^\s*//g; s/\s\s*/\t/g;' | \ - grep -w "^$1" | \ - cut -f2 -} - -pstree() { - for pid in $@; do - echo $pid - for child in $(pschildren $pid); do - pstree $child - done - done -} - -killtree() { - kill -9 $( - { set +x; } 2>/dev/null; - pstree $@; - set -x; - ) 2>/dev/null -} - -getshims() { - ps -e -o pid= -o args= | sed -e 's/^ *//; s/\s\s*/\t/;' | grep -w 'k3s/data/[^/]*/bin/containerd-shim' | cut -f1 -} - -killtree $({ set +x; } 2>/dev/null; getshims; set -x) - -do_unmount_and_remove() { - set +x - while read -r _ path _; do - case "$path" in $1*) echo "$path" ;; esac - done < /proc/self/mounts | sort -r | xargs -r -t -n 1 sh -c 'umount "$0" && rm -rf "$0"' - set -x -} - -do_unmount_and_remove '/run/k3s' -do_unmount_and_remove '/var/lib/rancher/k3s' -do_unmount_and_remove '/var/lib/kubelet/pods' -do_unmount_and_remove '/var/lib/kubelet/plugins' -do_unmount_and_remove '/run/netns/cni-' - -# Remove CNI namespaces -ip netns show 2>/dev/null | grep cni- | xargs -r -t -n 1 ip netns delete - -# Delete network interface(s) that match 'master cni0' -ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore; do - iface=${iface%%@*} - [ -z "$iface" ] || ip link delete $iface -done -ip link delete cni0 -ip link delete flannel.1 -ip link delete flannel-v6.1 -ip link delete kube-ipvs0 -ip link delete flannel-wg -ip link delete flannel-wg-v6 -ip link delete nodelocaldns -ip link delete cilium_host -ip link delete cilium_vxlan -ip -br link show | grep 'cali[a-f0-9]*' | awk -F '@' '{print $1}' | xargs -r -t -n 1 ip link delete -rm -rf /var/lib/cni/ -iptables-save | grep -v KUBE- | grep -v CNI- | grep -iv flannel | iptables-restore -ip6tables-save | grep -v KUBE- | grep -v CNI- | grep -iv flannel | ip6tables-restore - `))) diff --git a/cmd/kk/pkg/k3s/templates/uninstall_script.go b/cmd/kk/pkg/k3s/templates/uninstall_script.go deleted file mode 100644 index 797b86721..000000000 --- a/cmd/kk/pkg/k3s/templates/uninstall_script.go +++ /dev/null @@ -1,73 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -// K3sUninstallScript defines the template of k3s-killall script. -var K3sUninstallScript = template.Must(template.New("k3s-uninstall.sh").Parse( - dedent.Dedent(`#!/bin/sh -set -x -[ $(id -u) -eq 0 ] || exec sudo $0 $@ - -/usr/local/bin/k3s-killall.sh - -if which systemctl; then - systemctl disable k3s - systemctl reset-failed k3s - systemctl daemon-reload -fi -if which rc-update; then - rc-update delete k3s default -fi - -rm -f /etc/systemd/system/k3s.service -rm -rf /etc/systemd/system/k3s.service.d -rm -f /etc/systemd/system/k3s.service.env - -remove_uninstall() { - rm -f /usr/local/bin/k3s-uninstall.sh -} -trap remove_uninstall EXIT - -if (ls /etc/systemd/system/k3s*.service || ls /etc/init.d/k3s*) >/dev/null 2>&1; then - set +x; echo 'Additional k3s services installed, skipping uninstall of k3s'; set -x - exit -fi - -for cmd in kubectl crictl ctr; do - if [ -L /usr/local/bin/$cmd ]; then - rm -f /usr/local/bin/$cmd - fi -done - -rm -rf /etc/rancher/k3s -rm -rf /run/k3s -rm -rf /run/flannel -rm -rf /var/lib/rancher/k3s -rm -rf /var/lib/kubelet -rm -f /usr/local/bin/k3s -rm -f /usr/local/bin/k3s-killall.sh - -if type yum >/dev/null 2>&1; then - yum remove -y k3s-selinux - rm -f /etc/yum.repos.d/rancher-k3s-common*.repo -fi - `))) diff --git a/cmd/kk/pkg/k8e/k8e_status.go b/cmd/kk/pkg/k8e/k8e_status.go deleted file mode 100644 index 5315d0632..000000000 --- a/cmd/kk/pkg/k8e/k8e_status.go +++ /dev/null @@ -1,126 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package k8e - -import ( - "fmt" - "os" - "path/filepath" - "regexp" - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type K8eStatus struct { - Version string - ClusterInfo string - NodeToken string - KubeConfig string - NodesInfo map[string]string -} - -func NewK8eStatus() *K8eStatus { - return &K8eStatus{NodesInfo: make(map[string]string)} -} - -func (k *K8eStatus) SearchVersion(runtime connector.Runtime) error { - cmd := "/usr/local/bin/k8e --version | grep 'k8e' | awk '{print $3}'" - if output, err := runtime.GetRunner().Cmd(cmd, true); err != nil { - return errors.Wrap(errors.WithStack(err), "search current version failed") - } else { - k.Version = output - } - return nil -} - -func (k *K8eStatus) SearchKubeConfig(runtime connector.Runtime) error { - kubeCfgCmd := "cat /etc/k8e/k8e.yaml" - if kubeConfigStr, err := runtime.GetRunner().SudoCmd(kubeCfgCmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "search cluster kubeconfig failed") - } else { - k.KubeConfig = kubeConfigStr - } - return nil -} - -func (k *K8eStatus) SearchNodeToken(runtime connector.Runtime) error { - nodeTokenBase64Cmd := "cat /var/lib/k8e/server/node-token" - output, err := runtime.GetRunner().SudoCmd(nodeTokenBase64Cmd, true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get cluster node token failed") - } - k.NodeToken = output - return nil -} - -func (k *K8eStatus) SearchInfo(runtime connector.Runtime) error { - output, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl --no-headers=true get nodes -o custom-columns=:metadata.name,:status.nodeInfo.kubeletVersion,:status.addresses", - true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get K8e cluster info failed") - } - k.ClusterInfo = output - return nil -} - -func (k *K8eStatus) SearchNodesInfo(_ connector.Runtime) error { - ipv4Regexp, err := regexp.Compile(common.IPv4Regexp) - if err != nil { - return err - } - ipv6Regexp, err := regexp.Compile(common.IPv6Regexp) - if err != nil { - return err - } - tmp := strings.Split(k.ClusterInfo, "\r\n") - if len(tmp) < 1 || len(tmp) == 1 && tmp[0] == "" { - return errors.New("search K8e node info failed") - } - - for i := 0; i < len(tmp); i++ { - if ipv4 := ipv4Regexp.FindStringSubmatch(tmp[i]); len(ipv4) != 0 { - k.NodesInfo[ipv4[0]] = ipv4[0] - } - if ipv6 := ipv6Regexp.FindStringSubmatch(tmp[i]); len(ipv6) != 0 { - k.NodesInfo[ipv6[0]] = ipv6[0] - } - if len(strings.Fields(tmp[i])) > 3 { - k.NodesInfo[strings.Fields(tmp[i])[0]] = strings.Fields(tmp[i])[1] - } else { - k.NodesInfo[strings.Fields(tmp[i])[0]] = "" - } - } - return nil -} - -func (k *K8eStatus) LoadKubeConfig(runtime connector.Runtime, kubeConf *common.KubeConf) error { - kubeConfigPath := filepath.Join(runtime.GetWorkDir(), fmt.Sprintf("config-%s", runtime.GetObjName())) - - oldServer := "server: https://127.0.0.1:6443" - newServer := fmt.Sprintf("server: https://%s:%d", kubeConf.Cluster.ControlPlaneEndpoint.Address, kubeConf.Cluster.ControlPlaneEndpoint.Port) - newKubeConfigStr := strings.Replace(k.KubeConfig, oldServer, newServer, -1) - - if err := os.WriteFile(kubeConfigPath, []byte(newKubeConfigStr), 0644); err != nil { - return err - } - return nil -} diff --git a/cmd/kk/pkg/k8e/module.go b/cmd/kk/pkg/k8e/module.go deleted file mode 100644 index d5184852a..000000000 --- a/cmd/kk/pkg/k8e/module.go +++ /dev/null @@ -1,316 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package k8e - -import ( - "path/filepath" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/k8e/templates" -) - -type StatusModule struct { - common.KubeModule -} - -func (s *StatusModule) Init() { - s.Name = "StatusModule" - s.Desc = "Get cluster status" - - cluster := NewK8eStatus() - s.PipelineCache.GetOrSet(common.ClusterStatus, cluster) - - clusterStatus := &task.RemoteTask{ - Name: "GetClusterStatus", - Desc: "Get K8e cluster status", - Hosts: s.Runtime.GetHostsByRole(common.Master), - Action: new(GetClusterStatus), - Parallel: false, - } - - s.Tasks = []task.Interface{ - clusterStatus, - } -} - -type InstallKubeBinariesModule struct { - common.KubeModule -} - -func (i *InstallKubeBinariesModule) Init() { - i.Name = "InstallKubeBinariesModule" - i.Desc = "Install k8e cluster" - - syncBinary := &task.RemoteTask{ - Name: "SyncKubeBinary", - Desc: "Synchronize k8e binaries", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: new(SyncKubeBinary), - Parallel: true, - Retry: 2, - } - - killAllScript := &task.RemoteTask{ - Name: "GenerateK8eKillAllScript", - Desc: "Generate k8e killall.sh script", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: &action.Template{ - Template: templates.K8eKillallScript, - Dst: filepath.Join("/usr/local/bin", templates.K8eKillallScript.Name()), - }, - Parallel: true, - Retry: 2, - } - - uninstallScript := &task.RemoteTask{ - Name: "GenerateK8eUninstallScript", - Desc: "Generate k8e uninstall script", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: &action.Template{ - Template: templates.K8eUninstallScript, - Dst: filepath.Join("/usr/local/bin", templates.K8eUninstallScript.Name()), - }, - Parallel: true, - Retry: 2, - } - - chmod := &task.RemoteTask{ - Name: "ChmodScript", - Desc: "Chmod +x k8e script ", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: new(ChmodScript), - Parallel: true, - Retry: 2, - } - - i.Tasks = []task.Interface{ - syncBinary, - killAllScript, - uninstallScript, - chmod, - } -} - -type InitClusterModule struct { - common.KubeModule -} - -func (i *InitClusterModule) Init() { - i.Name = "K8eInitClusterModule" - i.Desc = "Init k8e cluster" - - k8eService := &task.RemoteTask{ - Name: "GenerateK8eService", - Desc: "Generate k8e Service", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: new(GenerateK8eService), - Parallel: true, - } - - k8eEnv := &task.RemoteTask{ - Name: "GenerateK8eServiceEnv", - Desc: "Generate k8e service env", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: new(GenerateK8eServiceEnv), - Parallel: true, - } - - enableK8e := &task.RemoteTask{ - Name: "EnableK8eService", - Desc: "Enable k8e service", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: new(EnableK8eService), - Parallel: true, - } - - copyKubeConfig := &task.RemoteTask{ - Name: "CopyKubeConfig", - Desc: "Copy k8e.yaml to ~/.kube/config", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: new(CopyK8eKubeConfig), - Parallel: true, - } - - addMasterTaint := &task.RemoteTask{ - Name: "AddMasterTaint", - Desc: "Add master taint", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - &common.IsWorker{Not: true}, - }, - Action: new(AddMasterTaint), - Parallel: true, - Retry: 5, - } - - i.Tasks = []task.Interface{ - k8eService, - k8eEnv, - enableK8e, - copyKubeConfig, - addMasterTaint, - } -} - -type JoinNodesModule struct { - common.KubeModule -} - -func (j *JoinNodesModule) Init() { - j.Name = "K8eJoinNodesModule" - j.Desc = "Join k8e nodes" - - k8eService := &task.RemoteTask{ - Name: "GenerateK8eService", - Desc: "Generate k8e Service", - Hosts: j.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - }, - Action: new(GenerateK8eService), - Parallel: true, - } - - k8eEnv := &task.RemoteTask{ - Name: "GenerateK8eServiceEnv", - Desc: "Generate k8e service env", - Hosts: j.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - }, - Action: new(GenerateK8eServiceEnv), - Parallel: true, - } - - enableK8e := &task.RemoteTask{ - Name: "EnableK8eService", - Desc: "Enable k8e service", - Hosts: j.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - }, - Action: new(EnableK8eService), - Parallel: true, - } - - copyKubeConfigForMaster := &task.RemoteTask{ - Name: "CopyKubeConfig", - Desc: "Copy k8e.yaml to ~/.kube/config", - Hosts: j.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - }, - Action: new(CopyK8eKubeConfig), - Parallel: true, - } - - addMasterTaint := &task.RemoteTask{ - Name: "AddMasterTaint", - Desc: "Add master taint", - Hosts: j.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - &common.IsWorker{Not: true}, - }, - Action: new(AddMasterTaint), - Parallel: true, - Retry: 5, - } - - addWorkerLabelToNode := &task.RemoteTask{ - Name: "addWorkerLabelToNode", - Desc: "Add worker label to all nodes", - Hosts: j.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(AddWorkerLabel), - Retry: 3, - } - - j.Tasks = []task.Interface{ - k8eService, - k8eEnv, - enableK8e, - copyKubeConfigForMaster, - addMasterTaint, - addWorkerLabelToNode, - } -} - -type DeleteClusterModule struct { - common.KubeModule -} - -func (d *DeleteClusterModule) Init() { - d.Name = "DeleteClusterModule" - d.Desc = "Delete k8e cluster" - - execScript := &task.RemoteTask{ - Name: "ExecUninstallScript", - Desc: "Exec k8e uninstall script", - Hosts: d.Runtime.GetHostsByRole(common.K8s), - Action: new(ExecUninstallScript), - Parallel: true, - } - - d.Tasks = []task.Interface{ - execScript, - } -} - -type SaveKubeConfigModule struct { - common.KubeModule -} - -func (s *SaveKubeConfigModule) Init() { - s.Name = "SaveKubeConfigModule" - s.Desc = "Save kube config file as a configmap" - - save := &task.LocalTask{ - Name: "SaveKubeConfig", - Desc: "Save kube config as a configmap", - Action: new(SaveKubeConfig), - } - - s.Tasks = []task.Interface{ - save, - } -} diff --git a/cmd/kk/pkg/k8e/prepares.go b/cmd/kk/pkg/k8e/prepares.go deleted file mode 100644 index 7a1d100a1..000000000 --- a/cmd/kk/pkg/k8e/prepares.go +++ /dev/null @@ -1,62 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package k8e - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type NodeInCluster struct { - common.KubePrepare - Not bool -} - -func (n *NodeInCluster) PreCheck(runtime connector.Runtime) (bool, error) { - host := runtime.RemoteHost() - if v, ok := n.PipelineCache.Get(common.ClusterStatus); ok { - cluster := v.(*K8eStatus) - var versionOk bool - if res, ok := cluster.NodesInfo[host.GetName()]; ok && res != "" { - versionOk = true - } - _, ipOk := cluster.NodesInfo[host.GetInternalAddress()] - if n.Not { - return !(versionOk || ipOk), nil - } - return versionOk || ipOk, nil - } - return false, errors.New("get k8e cluster status by pipeline cache failed") -} - -type ClusterIsExist struct { - common.KubePrepare - Not bool -} - -func (c *ClusterIsExist) PreCheck(_ connector.Runtime) (bool, error) { - if exist, ok := c.PipelineCache.GetMustBool(common.ClusterExist); ok { - if c.Not { - return !exist, nil - } - return exist, nil - } else { - return false, errors.New("get k8e cluster status by pipeline cache failed") - } -} diff --git a/cmd/kk/pkg/k8e/tasks.go b/cmd/kk/pkg/k8e/tasks.go deleted file mode 100644 index c1064dc4c..000000000 --- a/cmd/kk/pkg/k8e/tasks.go +++ /dev/null @@ -1,458 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package k8e - -import ( - "context" - "encoding/base64" - "fmt" - "path/filepath" - "strings" - - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - versionutil "k8s.io/apimachinery/pkg/util/version" - kube "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/k8e/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" -) - -type GetClusterStatus struct { - common.KubeAction -} - -func (g *GetClusterStatus) Execute(runtime connector.Runtime) error { - exist, err := runtime.GetRunner().FileExist("/etc/systemd/system/k8e.service") - if err != nil { - return err - } - - if !exist { - g.PipelineCache.Set(common.ClusterExist, false) - return nil - } else { - g.PipelineCache.Set(common.ClusterExist, true) - - if v, ok := g.PipelineCache.Get(common.ClusterStatus); ok { - cluster := v.(*K8eStatus) - if err := cluster.SearchVersion(runtime); err != nil { - return err - } - if err := cluster.SearchKubeConfig(runtime); err != nil { - return err - } - if err := cluster.LoadKubeConfig(runtime, g.KubeConf); err != nil { - return err - } - if err := cluster.SearchNodeToken(runtime); err != nil { - return err - } - if err := cluster.SearchInfo(runtime); err != nil { - return err - } - if err := cluster.SearchNodesInfo(runtime); err != nil { - return err - } - g.PipelineCache.Set(common.ClusterStatus, cluster) - } else { - return errors.New("get k8e cluster status by pipeline cache failed") - } - } - return nil -} - -type SyncKubeBinary struct { - common.KubeAction -} - -func (s *SyncKubeBinary) Execute(runtime connector.Runtime) error { - binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) - if !ok { - return errors.New("get KubeBinary by pipeline cache failed") - } - binariesMap := binariesMapObj.(map[string]*files.KubeBinary) - - if err := SyncKubeBinaries(runtime, binariesMap); err != nil { - return err - } - return nil -} - -// SyncKubeBinaries is used to sync kubernetes' binaries to each node. -func SyncKubeBinaries(runtime connector.Runtime, binariesMap map[string]*files.KubeBinary) error { - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - binaryList := []string{"k8e", "helm", "kubecni"} - for _, name := range binaryList { - binary, ok := binariesMap[name] - if !ok { - return fmt.Errorf("get kube binary %s info failed: no such key", name) - } - - fileName := binary.FileName - switch name { - case "kubecni": - dst := filepath.Join(common.TmpDir, fileName) - if err := runtime.GetRunner().Scp(binary.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync kube binaries failed")) - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("tar -zxf %s -C /opt/cni/bin", dst), false); err != nil { - return err - } - default: - dst := filepath.Join(common.BinDir, fileName) - if err := runtime.GetRunner().SudoScp(binary.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync kube binaries failed")) - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chmod +x %s", dst), false); err != nil { - return err - } - } - } - - binaries := []string{"kubectl", "crictl", "ctr"} - var createLinkCMDs []string - for _, binary := range binaries { - createLinkCMDs = append(createLinkCMDs, fmt.Sprintf("ln -snf /usr/local/bin/k8e /usr/local/bin/%s", binary)) - } - if _, err := runtime.GetRunner().SudoCmd(strings.Join(createLinkCMDs, " && "), false); err != nil { - return errors.Wrap(errors.WithStack(err), "create ctl tool link failed") - } - - return nil -} - -type ChmodScript struct { - common.KubeAction -} - -func (c *ChmodScript) Execute(runtime connector.Runtime) error { - killAllScript := filepath.Join("/usr/local/bin", templates.K8eKillallScript.Name()) - uninstallScript := filepath.Join("/usr/local/bin", templates.K8eUninstallScript.Name()) - - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chmod +x %s", killAllScript), - false); err != nil { - return err - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chmod +x %s", uninstallScript), - false); err != nil { - return err - } - return nil -} - -type GenerateK8eService struct { - common.KubeAction -} - -func (g *GenerateK8eService) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - - var server string - if !host.IsRole(common.Master) { - server = fmt.Sprintf("https://%s:%d", g.KubeConf.Cluster.ControlPlaneEndpoint.Domain, g.KubeConf.Cluster.ControlPlaneEndpoint.Port) - } - - defaultKubeletArs := map[string]string{ - "kube-reserved": "cpu=200m,memory=250Mi,ephemeral-storage=1Gi", - "system-reserved": "cpu=200m,memory=250Mi,ephemeral-storage=1Gi", - "eviction-hard": "memory.available<5%,nodefs.available<10%", - } - defaultKubeProxyArgs := map[string]string{ - "proxy-mode": "ipvs", - } - - kubeApiserverArgs, _ := util.GetArgs(map[string]string{}, g.KubeConf.Cluster.Kubernetes.ApiServerArgs) - kubeControllerManager, _ := util.GetArgs(map[string]string{ - "pod-eviction-timeout": "3m0s", - "terminated-pod-gc-threshold": "5", - }, g.KubeConf.Cluster.Kubernetes.ControllerManagerArgs) - kubeSchedulerArgs, _ := util.GetArgs(map[string]string{}, g.KubeConf.Cluster.Kubernetes.SchedulerArgs) - kubeletArgs, _ := util.GetArgs(defaultKubeletArs, g.KubeConf.Cluster.Kubernetes.KubeletArgs) - kubeProxyArgs, _ := util.GetArgs(defaultKubeProxyArgs, g.KubeConf.Cluster.Kubernetes.KubeProxyArgs) - - templateAction := action.Template{ - Template: templates.K8eService, - Dst: filepath.Join("/etc/systemd/system/", templates.K8eService.Name()), - Data: util.Data{ - "Server": server, - "IsMaster": host.IsRole(common.Master), - "NodeIP": host.GetInternalAddress(), - "HostName": host.GetName(), - "PodSubnet": g.KubeConf.Cluster.Network.KubePodsCIDR, - "ServiceSubnet": g.KubeConf.Cluster.Network.KubeServiceCIDR, - "ClusterDns": g.KubeConf.Cluster.CorednsClusterIP(), - "CertSANs": g.KubeConf.Cluster.GenerateCertSANs(), - "PauseImage": images.GetImage(runtime, g.KubeConf, "pause").ImageName(), - "ApiserverArgs": kubeApiserverArgs, - "ControllerManager": kubeControllerManager, - "SchedulerArgs": kubeSchedulerArgs, - "KubeletArgs": kubeletArgs, - "KubeProxyArgs": kubeProxyArgs, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type GenerateK8eServiceEnv struct { - common.KubeAction -} - -func (g *GenerateK8eServiceEnv) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - - clusterStatus, ok := g.PipelineCache.Get(common.ClusterStatus) - if !ok { - return errors.New("get cluster status by pipeline cache failed") - } - cluster := clusterStatus.(*K8eStatus) - - var externalEtcd kubekeyapiv1alpha2.ExternalEtcd - var endpointsList []string - var externalEtcdEndpoints, token string - - switch g.KubeConf.Cluster.Etcd.Type { - case kubekeyapiv1alpha2.External: - externalEtcd.Endpoints = g.KubeConf.Cluster.Etcd.External.Endpoints - - if len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 && len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 && len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 { - externalEtcd.CAFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.CAFile)) - externalEtcd.CertFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.CertFile)) - externalEtcd.KeyFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.KeyFile)) - } - default: - for _, node := range runtime.GetHostsByRole(common.ETCD) { - endpoint := fmt.Sprintf("https://%s:%s", node.GetInternalAddress(), kubekeyapiv1alpha2.DefaultEtcdPort) - endpointsList = append(endpointsList, endpoint) - } - externalEtcd.Endpoints = endpointsList - - externalEtcd.CAFile = "/etc/ssl/etcd/ssl/ca.pem" - externalEtcd.CertFile = fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s.pem", runtime.GetHostsByRole(common.Master)[0].GetName()) - externalEtcd.KeyFile = fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s-key.pem", runtime.GetHostsByRole(common.Master)[0].GetName()) - } - - externalEtcdEndpoints = strings.Join(externalEtcd.Endpoints, ",") - - v121 := versionutil.MustParseSemantic("v1.21.0") - atLeast := versionutil.MustParseSemantic(g.KubeConf.Cluster.Kubernetes.Version).AtLeast(v121) - if atLeast { - token = cluster.NodeToken - } else { - if !host.IsRole(common.Master) { - token = cluster.NodeToken - } - } - - templateAction := action.Template{ - Template: templates.K8eServiceEnv, - Dst: filepath.Join("/etc/systemd/system/", templates.K8eServiceEnv.Name()), - Data: util.Data{ - "DataStoreEndPoint": externalEtcdEndpoints, - "DataStoreCaFile": externalEtcd.CAFile, - "DataStoreCertFile": externalEtcd.CertFile, - "DataStoreKeyFile": externalEtcd.KeyFile, - "IsMaster": host.IsRole(common.Master), - "Token": token, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type EnableK8eService struct { - common.KubeAction -} - -func (e *EnableK8eService) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && systemctl enable --now k8e", - false); err != nil { - return errors.Wrap(errors.WithStack(err), "enable k8e failed") - } - return nil -} - -type CopyK8eKubeConfig struct { - common.KubeAction -} - -func (c *CopyK8eKubeConfig) Execute(runtime connector.Runtime) error { - createConfigDirCmd := "mkdir -p /root/.kube && mkdir -p $HOME/.kube" - getKubeConfigCmd := "cp -f /etc/k8e/k8e.yaml /root/.kube/config" - - cmd := strings.Join([]string{createConfigDirCmd, getKubeConfigCmd}, " && ") - if _, err := runtime.GetRunner().SudoCmd(cmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "copy k8e kube config failed") - } - - userMkdir := "mkdir -p $HOME/.kube" - if _, err := runtime.GetRunner().Cmd(userMkdir, false); err != nil { - return errors.Wrap(errors.WithStack(err), "user mkdir $HOME/.kube failed") - } - - userCopyKubeConfig := "cp -f /etc/k8e/k8e.yaml $HOME/.kube/config" - if _, err := runtime.GetRunner().SudoCmd(userCopyKubeConfig, false); err != nil { - return errors.Wrap(errors.WithStack(err), "user copy /etc/k8e/k8e.yaml to $HOME/.kube/config failed") - } - - userId, err := runtime.GetRunner().Cmd("echo $(id -u)", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get user id failed") - } - - userGroupId, err := runtime.GetRunner().Cmd("echo $(id -g)", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get user group id failed") - } - - chownKubeConfig := fmt.Sprintf("chown -R %s:%s $HOME/.kube", userId, userGroupId) - if _, err := runtime.GetRunner().SudoCmd(chownKubeConfig, false); err != nil { - return errors.Wrap(errors.WithStack(err), "chown user kube config failed") - } - return nil -} - -type AddMasterTaint struct { - common.KubeAction -} - -func (a *AddMasterTaint) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - - cmd := fmt.Sprintf( - "/usr/local/bin/kubectl taint nodes %s node-role.kubernetes.io/master=effect:NoSchedule --overwrite", - host.GetName()) - - if _, err := runtime.GetRunner().SudoCmd(cmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "add master NoSchedule taint failed") - } - return nil -} - -type AddWorkerLabel struct { - common.KubeAction -} - -func (a *AddWorkerLabel) Execute(runtime connector.Runtime) error { - for _, host := range runtime.GetAllHosts() { - if host.IsRole(common.Worker) { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "/usr/local/bin/kubectl label --overwrite node %s node-role.kubernetes.io/worker=", - host.GetName()), true); err != nil { - return errors.Wrap(errors.WithStack(err), "add worker label failed") - } - } - } - - return nil -} - -type ExecUninstallScript struct { - common.KubeAction -} - -func (e *ExecUninstallScript) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && /usr/local/bin/k8e-killall.sh", - true); err != nil { - return errors.Wrap(errors.WithStack(err), "add master NoSchedule taint failed") - } - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && /usr/local/bin/k8e-uninstall.sh", - true); err != nil { - return errors.Wrap(errors.WithStack(err), "add master NoSchedule taint failed") - } - return nil -} - -type SaveKubeConfig struct { - common.KubeAction -} - -func (s *SaveKubeConfig) Execute(_ connector.Runtime) error { - status, ok := s.PipelineCache.Get(common.ClusterStatus) - if !ok { - return errors.New("get kubernetes status failed by pipeline cache") - } - cluster := status.(*K8eStatus) - - oldServer := fmt.Sprintf("https://%s:%d", s.KubeConf.Cluster.ControlPlaneEndpoint.Domain, s.KubeConf.Cluster.ControlPlaneEndpoint.Port) - newServer := fmt.Sprintf("https://%s:%d", s.KubeConf.Cluster.ControlPlaneEndpoint.Address, s.KubeConf.Cluster.ControlPlaneEndpoint.Port) - newKubeConfigStr := strings.Replace(cluster.KubeConfig, oldServer, newServer, -1) - kubeConfigBase64 := base64.StdEncoding.EncodeToString([]byte(newKubeConfigStr)) - - config, err := clientcmd.NewClientConfigFromBytes([]byte(newKubeConfigStr)) - if err != nil { - return err - } - restConfig, err := config.ClientConfig() - if err != nil { - return err - } - clientsetForCluster, err := kube.NewForConfig(restConfig) - if err != nil { - return err - } - - namespace := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kubekey-system", - }, - } - if _, err := clientsetForCluster. - CoreV1(). - Namespaces(). - Create(context.TODO(), namespace, metav1.CreateOptions{}); err != nil { - return err - } - - cm := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-kubeconfig", s.KubeConf.ClusterName), - }, - Data: map[string]string{ - "kubeconfig": kubeConfigBase64, - }, - } - - if _, err := clientsetForCluster. - CoreV1(). - ConfigMaps("kubekey-system"). - Create(context.TODO(), cm, metav1.CreateOptions{}); err != nil { - return err - } - return nil -} diff --git a/cmd/kk/pkg/k8e/templates/k8eService.go b/cmd/kk/pkg/k8e/templates/k8eService.go deleted file mode 100644 index 3f974bd9a..000000000 --- a/cmd/kk/pkg/k8e/templates/k8eService.go +++ /dev/null @@ -1,59 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -var ( - // K8eService defines the template of kubelet service for systemd. - K8eService = template.Must(template.New("k8e.service").Parse( - dedent.Dedent(`[Unit] -Description=Simple Kubernetes Distribution -Documentation=https://getk8e.com -Wants=network-online.target -After=network-online.target - -[Install] -WantedBy=multi-user.target - -[Service] -Type=notify -EnvironmentFile=/etc/systemd/system/k8e.service.env -{{ if .IsMaster }} -Environment="K8E_ARGS= {{ range .CertSANs }} --tls-san={{ . }}{{- end }} {{ range .ApiserverArgs }} --kube-apiserver-arg={{ . }}{{- end }} {{ range .ControllerManager }} --kube-controller-manager-arg={{ . }}{{- end }} {{ range .SchedulerArgs }} --kube-scheduler-arg={{ . }}{{- end }} --cluster-cidr={{ .PodSubnet }} --service-cidr={{ .ServiceSubnet }} --cluster-dns={{ .ClusterDns }} " -{{ end }} -Environment="K8E_EXTRA_ARGS=--node-name={{ .HostName }} --node-ip={{ .NodeIP }} --pause-image={{ .PauseImage }} {{ range .KubeletArgs }} --kubelet-arg={{ . }}{{- end }} {{ range .KubeProxyArgs }} --kube-proxy-arg={{ . }}{{- end }}" -Environment="K8E_ROLE={{ if .IsMaster }}server{{ else }}agent{{ end }}" -Environment="K8E_SERVER_ARGS={{ if .Server }}--server={{ .Server }}{{ end }}" -KillMode=process -Delegate=yes -LimitNOFILE=1048576 -LimitNPROC=infinity -LimitCORE=infinity -TasksMax=infinity -TimeoutStartSec=0 -Restart=always -RestartSec=5s -ExecStartPre=-/sbin/modprobe br_netfilter -ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/local/bin/k8e $K8E_ROLE $K8E_ARGS $K8E_EXTRA_ARGS $K8E_SERVER_ARGS - `))) -) diff --git a/cmd/kk/pkg/k8e/templates/k8eServiceEnv.go b/cmd/kk/pkg/k8e/templates/k8eServiceEnv.go deleted file mode 100644 index e4985c288..000000000 --- a/cmd/kk/pkg/k8e/templates/k8eServiceEnv.go +++ /dev/null @@ -1,45 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -// K8eServiceEnv defines the template of kubelet's Env for the kubelet's systemd service. -var K8eServiceEnv = template.Must(template.New("k8e.service.env").Parse( - dedent.Dedent(`# Note: This dropin only works with k3s -{{ if .IsMaster }} -K8E_DATASTORE_ENDPOINT={{ .DataStoreEndPoint }} -{{- if .DataStoreCaFile }} -K8E_DATASTORE_CAFILE={{ .DataStoreCaFile }} -{{- end }} -{{- if .DataStoreCertFile }} -K8E_DATASTORE_CERTFILE={{ .DataStoreCertFile }} -{{- end }} -{{- if .DataStoreKeyFile }} -K8E_DATASTORE_KEYFILE={{ .DataStoreKeyFile }} -{{- end }} -K8E_KUBECONFIG_MODE=644 -{{ end }} -{{ if .Token }} -K8E_TOKEN={{ .Token }} -{{ end }} - - `))) diff --git a/cmd/kk/pkg/k8e/templates/killall_script.go b/cmd/kk/pkg/k8e/templates/killall_script.go deleted file mode 100644 index 9ae34db78..000000000 --- a/cmd/kk/pkg/k8e/templates/killall_script.go +++ /dev/null @@ -1,73 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -// K8eKillallScript defines the template of k3s-killall script. -var K8eKillallScript = template.Must(template.New("k8e-killall.sh").Parse( - dedent.Dedent(`#!/bin/sh -[ $(id -u) -eq 0 ] || exec sudo $0 $@ - -for service in /etc/systemd/system/k8e*.service; do - [ -s $service ] && systemctl stop $(basename $service) -done -pschildren() { - ps -e -o ppid= -o pid= | \ - sed -e 's/^\s*//g; s/\s\s*/\t/g;' | \ - grep -w "^$1" | \ - cut -f2 -} -pstree() { - for pid in $@; do - echo $pid - for child in $(pschildren $pid); do - pstree $child - done - done -} -killtree() { - kill -9 $( - { set +x; } 2>/dev/null; - pstree $@; - set -x; - ) 2>/dev/null -} -getshims() { - ps -e -o pid= -o args= | sed -e 's/^ *//; s/\s\s*/\t/;' | grep -w 'k8e/data/[^/]*/bin/containerd-shim' | cut -f1 -} -killtree $({ set +x; } 2>/dev/null; getshims; set -x) -do_unmount_and_remove() { - set +x - while read -r _ path _; do - case "$path" in $1*) echo "$path" ;; esac - done < /proc/self/mounts | sort -r | xargs -r -t -n 1 sh -c 'umount "$0" && rm -rf "$0"' - set -x -} -do_unmount_and_remove '/run/k8e' -do_unmount_and_remove '/var/lib/k8e' -do_unmount_and_remove '/var/lib/kubelet/pods' -do_unmount_and_remove '/var/lib/kubelet/plugins' -do_unmount_and_remove '/run/netns/cni-' -# Remove CNI namespaces -ip netns show 2>/dev/null | grep cni- | xargs -r -t -n 1 ip netns delete -rm -rf /var/lib/cni/ - `))) diff --git a/cmd/kk/pkg/k8e/templates/uninstall_script.go b/cmd/kk/pkg/k8e/templates/uninstall_script.go deleted file mode 100644 index dc793a99f..000000000 --- a/cmd/kk/pkg/k8e/templates/uninstall_script.go +++ /dev/null @@ -1,68 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -// K8eUninstallScript defines the template of k3s-killall script. -var K8eUninstallScript = template.Must(template.New("k8e-uninstall.sh").Parse( - dedent.Dedent(`#!/bin/sh -set -x -[ $(id -u) -eq 0 ] || exec sudo $0 $@ - -/usr/local/bin/k8e-killall.sh - -if which systemctl; then - systemctl disable k8e - systemctl reset-failed k8e - systemctl daemon-reload -fi -if which rc-update; then - rc-update delete k8e default -fi - -rm -f /etc/systemd/system/k8e.service -rm -rf /etc/systemd/system/k8e.service.d -rm -f /etc/systemd/system/k8e.service.env - -remove_uninstall() { - rm -f /usr/local/bin/k8e-uninstall.sh -} -trap remove_uninstall EXIT - -if (ls /etc/systemd/system/k8e*.service || ls /etc/init.d/k8e*) >/dev/null 2>&1; then - set +x; echo 'Additional k8e services installed, skipping uninstall of k8e'; set -x - exit -fi - -for cmd in kubectl crictl ctr; do - if [ -L /usr/local/bin/$cmd ]; then - rm -f /usr/local/bin/$cmd - fi -done - -rm -rf /etc/rancher/k8e -rm -rf /run/k8e -rm -rf /var/lib/k8e -rm -rf /var/lib/kubelet -rm -f /usr/local/bin/k8e -rm -f /usr/local/bin/k8e-killall.sh - `))) diff --git a/cmd/kk/pkg/kubernetes/kubernetes_status.go b/cmd/kk/pkg/kubernetes/kubernetes_status.go deleted file mode 100644 index 582d5f596..000000000 --- a/cmd/kk/pkg/kubernetes/kubernetes_status.go +++ /dev/null @@ -1,169 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubernetes - -import ( - "fmt" - "os" - "path/filepath" - "regexp" - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type KubernetesStatus struct { - Version string - BootstrapToken string - CertificateKey string - ClusterInfo string - KubeConfig string - NodesInfo map[string]string -} - -func NewKubernetesStatus() *KubernetesStatus { - return &KubernetesStatus{NodesInfo: make(map[string]string)} -} - -func (k *KubernetesStatus) SearchVersion(runtime connector.Runtime) error { - cmd := "sudo cat /etc/kubernetes/manifests/kube-apiserver.yaml | grep 'image:' | awk -F '[:]' '{print $(NF-0)}'" - if output, err := runtime.GetRunner().Cmd(cmd, true); err != nil { - return errors.Wrap(errors.WithStack(err), "search current version failed") - } else { - if !strings.Contains(output, "No such file or directory") { - k.Version = output - } - } - return nil -} - -func (k *KubernetesStatus) SearchJoinInfo(runtime connector.Runtime) error { - checkKubeadmConfig, err := runtime.GetRunner().SudoCmd("cat /etc/kubernetes/kubeadm-config.yaml", false) - if err != nil { - return err - } - if (k.BootstrapToken != "" || k.CertificateKey != "") && - (!strings.Contains(checkKubeadmConfig, "InitConfiguration") || - !strings.Contains(checkKubeadmConfig, "ClusterConfiguration")) { - return nil - } - - uploadCertsCmd := "/usr/local/bin/kubeadm init phase upload-certs --upload-certs --config /etc/kubernetes/kubeadm-config.yaml" - output, err := runtime.GetRunner().SudoCmd(uploadCertsCmd, true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "Failed to upload kubeadm certs") - } - reg := regexp.MustCompile("[0-9|a-z]{64}") - k.CertificateKey = reg.FindAllString(output, -1)[0] - - if err := patchKubeadmSecret(runtime); err != nil { - return err - } - - tokenCreateMasterCmd := "/usr/local/bin/kubeadm token create" - token, err := runtime.GetRunner().SudoCmd(tokenCreateMasterCmd, true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "Failed to get join node cmd") - } - - reg = regexp.MustCompile("[0-9|a-z]{6}.[0-9|a-z]{16}") - k.BootstrapToken = reg.FindAllString(token, -1)[0] - return nil -} - -func (k *KubernetesStatus) SearchClusterInfo(runtime connector.Runtime) error { - output, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl --no-headers=true get nodes -o custom-columns=:metadata.name,:status.nodeInfo.kubeletVersion,:status.addresses", - true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get kubernetes cluster info failed") - } - k.ClusterInfo = output - return nil -} - -func (k *KubernetesStatus) SearchNodesInfo(_ connector.Runtime) error { - ipv4Regexp, err := regexp.Compile(common.IPv4Regexp) - if err != nil { - return err - } - ipv6Regexp, err := regexp.Compile(common.IPv6Regexp) - if err != nil { - return err - } - tmp := strings.Split(k.ClusterInfo, "\r\n") - if len(tmp) >= 1 { - for i := 0; i < len(tmp); i++ { - if ipv4 := ipv4Regexp.FindStringSubmatch(tmp[i]); len(ipv4) != 0 { - k.NodesInfo[ipv4[0]] = ipv4[0] - } - if ipv6 := ipv6Regexp.FindStringSubmatch(tmp[i]); len(ipv6) != 0 { - k.NodesInfo[ipv6[0]] = ipv6[0] - } - if len(strings.Fields(tmp[i])) > 3 { - k.NodesInfo[strings.Fields(tmp[i])[0]] = strings.Fields(tmp[i])[1] - } else { - k.NodesInfo[strings.Fields(tmp[i])[0]] = "" - } - } - } - return nil -} - -func (k *KubernetesStatus) SearchKubeConfig(runtime connector.Runtime) error { - kubeCfgCmd := "cat /etc/kubernetes/admin.conf" - if kubeConfigStr, err := runtime.GetRunner().SudoCmd(kubeCfgCmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "search cluster kubeconfig failed") - } else { - k.KubeConfig = kubeConfigStr - } - return nil -} - -func (k *KubernetesStatus) LoadKubeConfig(runtime connector.Runtime, kubeConf *common.KubeConf) error { - kubeConfigPath := filepath.Join(runtime.GetWorkDir(), fmt.Sprintf("config-%s", runtime.GetObjName())) - kubeConfigStr := k.KubeConfig - - oldServer := fmt.Sprintf("server: https://%s:%d", kubeConf.Cluster.ControlPlaneEndpoint.Domain, kubeConf.Cluster.ControlPlaneEndpoint.Port) - if kubeConf.Cluster.ControlPlaneEndpoint.Address == "" { - kubeConf.Cluster.ControlPlaneEndpoint.Address = runtime.GetHostsByRole(common.Master)[0].GetAddress() - } - newServer := fmt.Sprintf("server: https://%s:%d", kubeConf.Cluster.ControlPlaneEndpoint.Address, kubeConf.Cluster.ControlPlaneEndpoint.Port) - newKubeConfigStr := strings.Replace(kubeConfigStr, oldServer, newServer, -1) - - if err := os.WriteFile(kubeConfigPath, []byte(newKubeConfigStr), 0644); err != nil { - return err - } - return nil -} - -// PatchKubeadmSecret is used to patch etcd's certs for kubeadm-certs secret. -func patchKubeadmSecret(runtime connector.Runtime) error { - externalEtcdCerts := []string{"external-etcd-ca.crt", "external-etcd.crt", "external-etcd.key"} - for _, cert := range externalEtcdCerts { - _, err := runtime.GetRunner().SudoCmd( - fmt.Sprintf("/usr/local/bin/kubectl patch -n kube-system secret kubeadm-certs -p '{\\\"data\\\": {\\\"%s\\\": \\\"\\\"}}'", cert), - true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "patch kubeadm secret failed") - } - } - return nil -} diff --git a/cmd/kk/pkg/kubernetes/module.go b/cmd/kk/pkg/kubernetes/module.go deleted file mode 100644 index c948c1c03..000000000 --- a/cmd/kk/pkg/kubernetes/module.go +++ /dev/null @@ -1,754 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubernetes - -import ( - "fmt" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/dns" - "path/filepath" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/binaries" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes/templates" - dnsTemplates "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/dns/templates" -) - -type StatusModule struct { - common.KubeModule -} - -func (k *StatusModule) Init() { - k.Name = "KubernetesStatusModule" - k.Desc = "Get kubernetes cluster status" - - cluster := NewKubernetesStatus() - k.PipelineCache.GetOrSet(common.ClusterStatus, cluster) - - clusterStatus := &task.RemoteTask{ - Name: "GetClusterStatus", - Desc: "Get kubernetes cluster status", - Hosts: k.Runtime.GetHostsByRole(common.Master), - //Prepare: new(NoClusterInfo), - Action: new(GetClusterStatus), - Parallel: false, - } - - k.Tasks = []task.Interface{ - clusterStatus, - } -} - -type InstallKubeBinariesModule struct { - common.KubeModule -} - -func (i *InstallKubeBinariesModule) Init() { - i.Name = "InstallKubeBinariesModule" - i.Desc = "Install kubernetes cluster" - - syncBinary := &task.RemoteTask{ - Name: "SyncKubeBinary", - Desc: "Synchronize kubernetes binaries", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: new(SyncKubeBinary), - Parallel: true, - Retry: 2, - } - - chmodKubelet := &task.RemoteTask{ - Name: "ChmodKubelet", - Desc: "Change kubelet mode", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: new(ChmodKubelet), - Parallel: true, - Retry: 2, - } - - generateKubeletService := &task.RemoteTask{ - Name: "GenerateKubeletService", - Desc: "Generate kubelet service", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: &action.Template{ - Template: templates.KubeletService, - Dst: filepath.Join("/etc/systemd/system/", templates.KubeletService.Name()), - }, - Parallel: true, - Retry: 2, - } - - enableKubelet := &task.RemoteTask{ - Name: "EnableKubelet", - Desc: "Enable kubelet service", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: new(EnableKubelet), - Parallel: true, - Retry: 5, - } - - generateKubeletEnv := &task.RemoteTask{ - Name: "GenerateKubeletEnv", - Desc: "Generate kubelet env", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: new(GenerateKubeletEnv), - Parallel: true, - Retry: 2, - } - - i.Tasks = []task.Interface{ - syncBinary, - chmodKubelet, - generateKubeletService, - enableKubelet, - generateKubeletEnv, - } -} - -type InitKubernetesModule struct { - common.KubeModule -} - -func (i *InitKubernetesModule) Init() { - i.Name = "InitKubernetesModule" - i.Desc = "Init kubernetes cluster" - - generateKubeadmConfig := &task.RemoteTask{ - Name: "GenerateKubeadmConfig", - Desc: "Generate kubeadm config", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: &GenerateKubeadmConfig{ - IsInitConfiguration: true, - WithSecurityEnhancement: i.KubeConf.Arg.SecurityEnhancement, - }, - Parallel: true, - } - - generateAuditPolicy := &task.RemoteTask{ - Name: "GenerateAduitPolicy", - Desc: "Generate audit policy", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.EnableAudit), - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: &action.Template{ - Template: templates.AuditPolicy, - Dst: filepath.Join("/etc/kubernetes/audit", templates.AuditPolicy.Name()), - }, - Parallel: true, - Retry: 2, - } - - generateAuditWebhook := &task.RemoteTask{ - Name: "GenerateAduitWebhook", - Desc: "Generate audit webhook", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.EnableAudit), - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: &action.Template{ - Template: templates.AuditWebhook, - Dst: filepath.Join("/etc/kubernetes/audit", templates.AuditWebhook.Name()), - }, - Parallel: true, - Retry: 2, - } - - kubeadmInit := &task.RemoteTask{ - Name: "KubeadmInit", - Desc: "Init cluster using kubeadm", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: new(KubeadmInit), - Retry: 3, - Parallel: true, - } - - copyKubeConfig := &task.RemoteTask{ - Name: "CopyKubeConfig", - Desc: "Copy admin.conf to ~/.kube/config", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: new(CopyKubeConfigForControlPlane), - Parallel: true, - } - - removeMasterTaint := &task.RemoteTask{ - Name: "RemoveMasterTaint", - Desc: "Remove master taint", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - new(common.IsWorker), - }, - Action: new(RemoveMasterTaint), - Parallel: true, - Retry: 5, - } - - i.Tasks = []task.Interface{ - generateKubeadmConfig, - generateAuditPolicy, - generateAuditWebhook, - kubeadmInit, - copyKubeConfig, - removeMasterTaint, - } -} - -type JoinNodesModule struct { - common.KubeModule -} - -func (j *JoinNodesModule) Init() { - j.Name = "JoinNodesModule" - j.Desc = "Join kubernetes nodes" - - j.PipelineCache.Set(common.ClusterExist, true) - - generateKubeadmConfig := &task.RemoteTask{ - Name: "GenerateKubeadmConfig", - Desc: "Generate kubeadm config", - Hosts: j.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - }, - Action: &GenerateKubeadmConfig{ - IsInitConfiguration: false, - WithSecurityEnhancement: j.KubeConf.Arg.SecurityEnhancement, - }, - Parallel: true, - } - - generateAuditPolicy := &task.RemoteTask{ - Name: "GenerateAduitPolicy", - Desc: "Generate audit policy", - Hosts: j.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.EnableAudit), - &NodeInCluster{Not: true}, - }, - Action: &action.Template{ - Template: templates.AuditPolicy, - Dst: filepath.Join("/etc/kubernetes/audit", templates.AuditPolicy.Name()), - }, - Parallel: true, - Retry: 2, - } - - generateAuditWebhook := &task.RemoteTask{ - Name: "GenerateAduitWebhook", - Desc: "Generate audit webhook", - Hosts: j.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.EnableAudit), - &NodeInCluster{Not: true}, - }, - Action: &action.Template{ - Template: templates.AuditWebhook, - Dst: filepath.Join("/etc/kubernetes/audit", templates.AuditWebhook.Name()), - }, - Parallel: true, - Retry: 2, - } - - joinMasterNode := &task.RemoteTask{ - Name: "JoinControlPlaneNode", - Desc: "Join control-plane node", - Hosts: j.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - }, - Action: new(JoinNode), - Parallel: true, - Retry: 5, - } - - joinWorkerNode := &task.RemoteTask{ - Name: "JoinWorkerNode", - Desc: "Join worker node", - Hosts: j.Runtime.GetHostsByRole(common.Worker), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - new(common.OnlyWorker), - }, - Action: new(JoinNode), - Parallel: true, - Retry: 5, - } - - copyKubeConfig := &task.RemoteTask{ - Name: "copyKubeConfig", - Desc: "Copy admin.conf to ~/.kube/config", - Hosts: j.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - }, - Action: new(CopyKubeConfigForControlPlane), - Parallel: true, - Retry: 2, - } - - removeMasterTaint := &task.RemoteTask{ - Name: "RemoveMasterTaint", - Desc: "Remove master taint", - Hosts: j.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - new(common.IsWorker), - }, - Action: new(RemoveMasterTaint), - Parallel: true, - Retry: 5, - } - - addWorkerLabelToNode := &task.RemoteTask{ - Name: "addWorkerLabelToNode", - Desc: "Add worker label to all nodes", - Hosts: j.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(AddWorkerLabel), - Retry: 3, - } - - j.Tasks = []task.Interface{ - generateKubeadmConfig, - generateAuditPolicy, - generateAuditWebhook, - joinMasterNode, - joinWorkerNode, - copyKubeConfig, - removeMasterTaint, - addWorkerLabelToNode, - } -} - -type ResetClusterModule struct { - common.KubeModule -} - -func (r *ResetClusterModule) Init() { - r.Name = "ResetClusterModule" - r.Desc = "Reset kubernetes cluster" - - kubeadmReset := &task.RemoteTask{ - Name: "KubeadmReset", - Desc: "Reset the cluster using kubeadm", - Hosts: r.Runtime.GetHostsByRole(common.K8s), - Action: new(KubeadmReset), - Parallel: true, - } - - r.Tasks = []task.Interface{ - kubeadmReset, - } -} - -type CompareConfigAndClusterInfoModule struct { - common.KubeModule -} - -func (c *CompareConfigAndClusterInfoModule) Init() { - c.Name = "CompareConfigAndClusterInfoModule" - c.Desc = "Compare config and cluster nodes info" - - check := &task.RemoteTask{ - Name: "FindNode", - Desc: "Find information about nodes that are expected to be deleted", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - //Action: new(FindNode), - Action: new(FilterFirstMaster), - } - - c.Tasks = []task.Interface{ - check, - } -} - -type RestartKubeletModule struct { - common.KubeModule -} - -func (r *RestartKubeletModule) init() { - r.Name = "RestartKubeletModule" - r.Desc = "restart node kubelet service " - restart := &task.RemoteTask{ - Name: "RestartKubelet", - Desc: "Restart kubelet service", - Hosts: r.Runtime.GetHostsByRole(common.Master), - Action: new(RestartKubelet), - Retry: 5, - } - - r.Tasks = []task.Interface{ - restart, - } -} - -type DeleteKubeNodeModule struct { - common.KubeModule -} - -func (d *DeleteKubeNodeModule) Init() { - d.Name = "DeleteKubeNodeModule" - d.Desc = "Delete kubernetes node" - - drain := &task.RemoteTask{ - Name: "DrainNode", - Desc: "Node safely evict all pods", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(DrainNode), - Retry: 2, - } - - deleteNode := &task.RemoteTask{ - Name: "DeleteNode", - Desc: "Delete the node using kubectl", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(KubectlDeleteNode), - Retry: 5, - } - - d.Tasks = []task.Interface{ - drain, - deleteNode, - } -} - -type SetUpgradePlanModule struct { - common.KubeModule - Step UpgradeStep -} - -func (s *SetUpgradePlanModule) Init() { - s.Name = fmt.Sprintf("SetUpgradePlanModule %d/%d", s.Step, len(UpgradeStepList)) - s.Desc = "Set upgrade plan" - - plan := &task.LocalTask{ - Name: "SetUpgradePlan", - Desc: "Set upgrade plan", - Action: &SetUpgradePlan{Step: s.Step}, - } - - generateKubeadmConfigInit := &task.RemoteTask{ - Name: "GenerateKubeadmConfig", - Desc: "Generate kubeadm config", - Hosts: s.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - }, - Action: &GenerateKubeadmConfig{ - IsInitConfiguration: true, - WithSecurityEnhancement: s.KubeConf.Arg.SecurityEnhancement, - }, - Parallel: true, - } - - s.Tasks = []task.Interface{ - plan, - generateKubeadmConfigInit, - } -} - -type ProgressiveUpgradeModule struct { - common.KubeModule - Step UpgradeStep -} - -func (p *ProgressiveUpgradeModule) Init() { - p.Name = fmt.Sprintf("ProgressiveUpgradeModule %d/%d", p.Step, len(UpgradeStepList)) - p.Desc = fmt.Sprintf("Progressive upgrade %d/%d", p.Step, len(UpgradeStepList)) - - nextVersion := &task.LocalTask{ - Name: "CalculateNextVersion", - Desc: "Calculate next upgrade version", - Prepare: new(NotEqualPlanVersion), - Action: new(CalculateNextVersion), - } - - download := &task.LocalTask{ - Name: "DownloadBinaries", - Desc: "Download installation binaries", - Prepare: new(NotEqualPlanVersion), - Action: new(binaries.Download), - } - - pull := &task.RemoteTask{ - Name: "PullImages", - Desc: "Start to pull images on all nodes", - Hosts: p.Runtime.GetHostsByRole(common.K8s), - Prepare: new(NotEqualPlanVersion), - Action: new(images.PullImage), - Parallel: true, - } - - syncBinary := &task.RemoteTask{ - Name: "SyncKubeBinary", - Desc: "Synchronize kubernetes binaries", - Hosts: p.Runtime.GetHostsByRole(common.K8s), - Prepare: new(NotEqualPlanVersion), - Action: new(SyncKubeBinary), - Parallel: true, - Retry: 2, - } - - upgradeKubeMaster := &task.RemoteTask{ - Name: "UpgradeClusterOnMaster", - Desc: "Upgrade cluster on master", - Hosts: p.Runtime.GetHostsByRole(common.Master), - Prepare: new(NotEqualPlanVersion), - Action: &UpgradeKubeMaster{ModuleName: p.Name}, - Parallel: false, - } - - cluster := NewKubernetesStatus() - p.PipelineCache.GetOrSet(common.ClusterStatus, cluster) - - clusterStatus := &task.RemoteTask{ - Name: "GetClusterStatus", - Desc: "Get kubernetes cluster status", - Hosts: p.Runtime.GetHostsByRole(common.Master), - Prepare: new(NotEqualPlanVersion), - Action: new(GetClusterStatus), - Parallel: false, - } - - generateCoreDNS := &task.RemoteTask{ - Name: "GenerateCoreDNS", - Desc: "Generate coredns manifests", - Hosts: p.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - }, - Action: new(dns.GenerateCorednsmanifests), - Parallel: true, - } - - applyCoredns := &task.RemoteTask{ - Name: "DeployCoreDNS", - Desc: "Deploy coredns", - Hosts: p.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - }, - Action: new(dns.DeployCoreDNS), - Parallel: true, - } - - generateNodeLocalDNS := &task.RemoteTask{ - Name: "GenerateNodeLocalDNS", - Desc: "Generate nodelocaldns", - Hosts: p.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(dns.EnableNodeLocalDNS), - }, - Action: &action.Template{ - Template: dnsTemplates.NodeLocalDNSService, - Dst: filepath.Join(common.KubeConfigDir, dnsTemplates.NodeLocalDNSService.Name()), - Data: util.Data{ - "NodelocaldnsImage": images.GetImage(p.Runtime, p.KubeConf, "k8s-dns-node-cache").ImageName(), - "DNSEtcHosts": p.KubeConf.Cluster.DNS.DNSEtcHosts, - }, - }, - Parallel: true, - } - - applyNodeLocalDNS := &task.RemoteTask{ - Name: "DeployNodeLocalDNS", - Desc: "Deploy nodelocaldns", - Hosts: p.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(dns.EnableNodeLocalDNS)}, - Action: new(dns.DeployNodeLocalDNS), - Parallel: true, - Retry: 5, - } - - upgradeKubeWorker := &task.RemoteTask{ - Name: "UpgradeClusterOnWorker", - Desc: "Upgrade cluster on worker", - Hosts: p.Runtime.GetHostsByRole(common.Worker), - Prepare: &prepare.PrepareCollection{ - new(NotEqualPlanVersion), - new(common.OnlyWorker), - }, - Action: &UpgradeKubeWorker{ModuleName: p.Name}, - Parallel: false, - } - - currentVersion := &task.LocalTask{ - Name: "SetCurrentK8sVersion", - Desc: "Set current k8s version", - Prepare: new(NotEqualPlanVersion), - Action: new(SetCurrentK8sVersion), - } - - p.Tasks = []task.Interface{ - nextVersion, - download, - pull, - syncBinary, - upgradeKubeMaster, - clusterStatus, - upgradeKubeWorker, - generateCoreDNS, - applyCoredns, - generateNodeLocalDNS, - applyNodeLocalDNS, - currentVersion, - } -} - -func (p *ProgressiveUpgradeModule) Until() (*bool, error) { - f := false - t := true - currentVersion, ok := p.PipelineCache.GetMustString(common.K8sVersion) - if !ok { - return &f, errors.New("get current Kubernetes version failed by pipeline cache") - } - planVersion, ok := p.PipelineCache.GetMustString(common.PlanK8sVersion) - if !ok { - return &f, errors.New("get upgrade plan Kubernetes version failed by pipeline cache") - } - - if currentVersion != planVersion { - return &f, nil - } else { - originalDesired, ok := p.PipelineCache.GetMustString(common.DesiredK8sVersion) - if !ok { - return &f, errors.New("get original desired Kubernetes version failed by pipeline cache") - } - p.KubeConf.Cluster.Kubernetes.Version = originalDesired - return &t, nil - } -} - -type SaveKubeConfigModule struct { - common.KubeModule -} - -func (s *SaveKubeConfigModule) Init() { - s.Name = "SaveKubeConfigModule" - s.Desc = "Save kube config file as a configmap" - - save := &task.LocalTask{ - Name: "SaveKubeConfig", - Desc: "Save kube config as a configmap", - Action: new(SaveKubeConfig), - Retry: 5, - } - - s.Tasks = []task.Interface{ - save, - } -} - -type ConfigureKubernetesModule struct { - common.KubeModule -} - -func (c *ConfigureKubernetesModule) Init() { - c.Name = "ConfigureKubernetesModule" - c.Desc = "Configure kubernetes" - - configure := &task.RemoteTask{ - Name: "ConfigureKubernetes", - Desc: "Configure kubernetes", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(ConfigureKubernetes), - Retry: 3, - } - - c.Tasks = []task.Interface{ - configure, - } -} - -type SecurityEnhancementModule struct { - common.KubeModule - Skip bool -} - -func (s *SecurityEnhancementModule) IsSkip() bool { - return s.Skip -} - -func (s *SecurityEnhancementModule) Init() { - s.Name = "SecurityEnhancementModule" - s.Desc = "Security enhancement for the cluster" - - etcdSecurityEnhancement := &task.RemoteTask{ - Name: "EtcdSecurityEnhancementTask", - Desc: "Security enhancement for etcd", - Hosts: s.Runtime.GetHostsByRole(common.ETCD), - Action: new(EtcdSecurityEnhancemenAction), - Parallel: true, - } - - masterSecurityEnhancement := &task.RemoteTask{ - Name: "K8sSecurityEnhancementTask", - Desc: "Security enhancement for kubernetes", - Hosts: s.Runtime.GetHostsByRole(common.Master), - Action: new(MasterSecurityEnhancemenAction), - Parallel: true, - } - - nodesSecurityEnhancement := &task.RemoteTask{ - Name: "K8sSecurityEnhancementTask", - Desc: "Security enhancement for kubernetes", - Hosts: s.Runtime.GetHostsByRole(common.Worker), - Action: new(NodesSecurityEnhancemenAction), - Parallel: true, - } - - s.Tasks = []task.Interface{ - etcdSecurityEnhancement, - masterSecurityEnhancement, - nodesSecurityEnhancement, - } -} diff --git a/cmd/kk/pkg/kubernetes/prepares.go b/cmd/kk/pkg/kubernetes/prepares.go deleted file mode 100644 index c0f2e5300..000000000 --- a/cmd/kk/pkg/kubernetes/prepares.go +++ /dev/null @@ -1,133 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubernetes - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type NoClusterInfo struct { - common.KubePrepare -} - -func (n *NoClusterInfo) PreCheck(_ connector.Runtime) (bool, error) { - if v, ok := n.PipelineCache.Get(common.ClusterStatus); ok { - cluster := v.(*KubernetesStatus) - if cluster.ClusterInfo == "" { - return true, nil - } - } else { - return false, errors.New("get kubernetes cluster status by pipeline cache failed") - } - return false, nil -} - -type NodeInCluster struct { - common.KubePrepare - Not bool -} - -func (n *NodeInCluster) PreCheck(runtime connector.Runtime) (bool, error) { - host := runtime.RemoteHost() - if v, ok := n.PipelineCache.Get(common.ClusterStatus); ok { - cluster := v.(*KubernetesStatus) - var versionOk bool - if res, ok := cluster.NodesInfo[host.GetName()]; ok && res != "" { - versionOk = true - } - _, ipOk := cluster.NodesInfo[host.GetInternalAddress()] - if n.Not { - return !(versionOk || ipOk), nil - } - return versionOk || ipOk, nil - } else { - return false, errors.New("get kubernetes cluster status by pipeline cache failed") - } -} - -type ClusterIsExist struct { - common.KubePrepare - Not bool -} - -func (c *ClusterIsExist) PreCheck(_ connector.Runtime) (bool, error) { - if exist, ok := c.PipelineCache.GetMustBool(common.ClusterExist); ok { - if c.Not { - return !exist, nil - } - return exist, nil - } else { - return false, errors.New("get kubernetes cluster status by pipeline cache failed") - } -} - -type NotEqualPlanVersion struct { - common.KubePrepare -} - -func (n *NotEqualPlanVersion) PreCheck(runtime connector.Runtime) (bool, error) { - planVersion, ok := n.PipelineCache.GetMustString(common.PlanK8sVersion) - if !ok { - return false, errors.New("get upgrade plan Kubernetes version failed by pipeline cache") - } - - currentVersion, ok := n.PipelineCache.GetMustString(common.K8sVersion) - if !ok { - return false, errors.New("get cluster Kubernetes version failed by pipeline cache") - } - if currentVersion == planVersion { - return false, nil - } - return true, nil -} - -type ClusterNotEqualDesiredVersion struct { - common.KubePrepare -} - -func (c *ClusterNotEqualDesiredVersion) PreCheck(runtime connector.Runtime) (bool, error) { - clusterK8sVersion, ok := c.PipelineCache.GetMustString(common.K8sVersion) - if !ok { - return false, errors.New("get cluster Kubernetes version failed by pipeline cache") - } - - if c.KubeConf.Cluster.Kubernetes.Version == clusterK8sVersion { - return false, nil - } - return true, nil -} - -type NotEqualDesiredVersion struct { - common.KubePrepare -} - -func (n *NotEqualDesiredVersion) PreCheck(runtime connector.Runtime) (bool, error) { - host := runtime.RemoteHost() - - nodeK8sVersion, ok := host.GetCache().GetMustString(common.NodeK8sVersion) - if !ok { - return false, errors.New("get node Kubernetes version failed by host cache") - } - - if n.KubeConf.Cluster.Kubernetes.Version == nodeK8sVersion { - return false, nil - } - return true, nil -} diff --git a/cmd/kk/pkg/kubernetes/task_test.go b/cmd/kk/pkg/kubernetes/task_test.go deleted file mode 100644 index 8fdf21da7..000000000 --- a/cmd/kk/pkg/kubernetes/task_test.go +++ /dev/null @@ -1,73 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubernetes - -import ( - "testing" -) - -func Test_calculateNextStr(t *testing.T) { - tests := []struct { - currentVersion string - desiredVersion string - want string - wantErr bool - errMsg string - }{ - { - currentVersion: "v1.21.5", - desiredVersion: "v1.22.5", - want: "v1.22.5", - wantErr: false, - }, - { - currentVersion: "v1.21.5", - desiredVersion: "v1.23.5", - want: "v1.22.12", - wantErr: false, - }, - { - currentVersion: "v1.17.5", - desiredVersion: "v1.18.5", - want: "", - wantErr: true, - errMsg: "the target version v1.18.5 is not supported", - }, - { - currentVersion: "v1.17.5", - desiredVersion: "v1.21.5", - want: "", - wantErr: true, - errMsg: "Kubernetes minor version v1.18.x is not supported", - }, - } - for _, tt := range tests { - t.Run("", func(t *testing.T) { - got, err := calculateNextStr(tt.currentVersion, tt.desiredVersion) - if (err != nil) != tt.wantErr { - t.Errorf("calculateNextStr() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("calculateNextStr() got = %v, want %v", got, tt.want) - } - if err != nil && err.Error() != tt.errMsg { - t.Errorf("calculateNextStr() error = %v, want %v", err, tt.errMsg) - } - }) - } -} diff --git a/cmd/kk/pkg/kubernetes/tasks.go b/cmd/kk/pkg/kubernetes/tasks.go deleted file mode 100644 index cc14106e5..000000000 --- a/cmd/kk/pkg/kubernetes/tasks.go +++ /dev/null @@ -1,1240 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubernetes - -import ( - "context" - "encoding/base64" - "fmt" - "os" - "path/filepath" - "sort" - "strings" - "time" - - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - kubeerrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - versionutil "k8s.io/apimachinery/pkg/util/version" - kube "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - - kubekeyv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/etcd" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" -) - -type GetClusterStatus struct { - common.KubeAction -} - -func (g *GetClusterStatus) Execute(runtime connector.Runtime) error { - exist, err := runtime.GetRunner().FileExist("/etc/kubernetes/admin.conf") - if err != nil { - return err - } - - if !exist { - g.PipelineCache.Set(common.ClusterExist, false) - return nil - } else { - g.PipelineCache.Set(common.ClusterExist, true) - - if v, ok := g.PipelineCache.Get(common.ClusterStatus); ok { - cluster := v.(*KubernetesStatus) - if err := cluster.SearchVersion(runtime); err != nil { - return err - } - if err := cluster.SearchKubeConfig(runtime); err != nil { - return err - } - if err := cluster.LoadKubeConfig(runtime, g.KubeConf); err != nil { - return err - } - if err := cluster.SearchClusterInfo(runtime); err != nil { - return err - } - if err := cluster.SearchNodesInfo(runtime); err != nil { - return err - } - if err := cluster.SearchJoinInfo(runtime); err != nil { - return err - } - - g.PipelineCache.Set(common.ClusterStatus, cluster) - } else { - return errors.New("get kubernetes cluster status by pipeline cache failed") - } - } - return nil -} - -type SyncKubeBinary struct { - common.KubeAction -} - -func (i *SyncKubeBinary) Execute(runtime connector.Runtime) error { - binariesMapObj, ok := i.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) - if !ok { - return errors.New("get KubeBinary by pipeline cache failed") - } - binariesMap := binariesMapObj.(map[string]*files.KubeBinary) - - if err := SyncKubeBinaries(i, runtime, binariesMap); err != nil { - return err - } - return nil -} - -// SyncKubeBinaries is used to sync kubernetes' binaries to each node. -func SyncKubeBinaries(i *SyncKubeBinary, runtime connector.Runtime, binariesMap map[string]*files.KubeBinary) error { - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - binaryList := []string{"kubeadm", "kubelet", "kubectl", "helm", "kubecni"} - if i.KubeConf.Cluster.Network.Plugin == "calico" { - binaryList = append(binaryList, "calicoctl") - } - for _, name := range binaryList { - binary, ok := binariesMap[name] - if !ok { - return fmt.Errorf("get kube binary %s info failed: no such key", name) - } - - fileName := binary.FileName - switch name { - //case "kubelet": - // if err := runtime.GetRunner().Scp(binary.Path, fmt.Sprintf("%s/%s", common.TmpDir, binary.Name)); err != nil { - // return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync kube binaries failed")) - // } - case "kubecni": - dst := filepath.Join(common.TmpDir, fileName) - if err := runtime.GetRunner().Scp(binary.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync kube binaries failed")) - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("tar -zxf %s -C /opt/cni/bin", dst), false); err != nil { - return err - } - default: - dst := filepath.Join(common.BinDir, fileName) - if err := runtime.GetRunner().SudoScp(binary.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync kube binaries failed")) - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chmod +x %s", dst), false); err != nil { - return err - } - } - } - return nil -} - -type ChmodKubelet struct { - common.KubeAction -} - -func (c *ChmodKubelet) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("chmod +x /usr/local/bin/kubelet", false); err != nil { - return errors.Wrap(errors.WithStack(err), "change kubelet mode failed") - } - return nil -} - -type EnableKubelet struct { - common.KubeAction -} - -func (e *EnableKubelet) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("systemctl disable kubelet "+ - "&& systemctl enable kubelet "+ - "&& ln -snf /usr/local/bin/kubelet /usr/bin/kubelet", false); err != nil { - return errors.Wrap(errors.WithStack(err), "enable kubelet service failed") - } - return nil -} - -type GenerateKubeletEnv struct { - common.KubeAction -} - -func (g *GenerateKubeletEnv) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - templateAction := action.Template{ - Template: templates.KubeletEnv, - Dst: filepath.Join("/etc/systemd/system/kubelet.service.d", templates.KubeletEnv.Name()), - Data: util.Data{ - "NodeIP": host.GetInternalAddress(), - "Hostname": host.GetName(), - "ContainerRuntime": "", - "KubeletArgs": g.KubeConf.Cluster.Kubernetes.KubeletArgs, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type GenerateKubeadmConfig struct { - common.KubeAction - IsInitConfiguration bool - WithSecurityEnhancement bool -} - -func (g *GenerateKubeadmConfig) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - - localConfig := filepath.Join(runtime.GetWorkDir(), "kubeadm-config.yaml") - if util.IsExist(localConfig) { - // todo: if it is necessary? - if err := runtime.GetRunner().SudoScp(localConfig, "/etc/kubernetes/kubeadm-config.yaml"); err != nil { - return errors.Wrap(errors.WithStack(err), "scp local kubeadm config failed") - } - } else { - // generate etcd configuration - var externalEtcd kubekeyv1alpha2.ExternalEtcd - var endpointsList, etcdCertSANs []string - - switch g.KubeConf.Cluster.Etcd.Type { - case kubekeyv1alpha2.KubeKey: - for _, host := range runtime.GetHostsByRole(common.ETCD) { - endpoint := fmt.Sprintf("https://%s:%s", host.GetInternalAddress(), kubekeyv1alpha2.DefaultEtcdPort) - endpointsList = append(endpointsList, endpoint) - } - externalEtcd.Endpoints = endpointsList - - externalEtcd.CAFile = "/etc/ssl/etcd/ssl/ca.pem" - externalEtcd.CertFile = fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s.pem", host.GetName()) - externalEtcd.KeyFile = fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s-key.pem", host.GetName()) - case kubekeyv1alpha2.External: - externalEtcd.Endpoints = g.KubeConf.Cluster.Etcd.External.Endpoints - - if len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 && len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 && len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 { - externalEtcd.CAFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.CAFile)) - externalEtcd.CertFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.CertFile)) - externalEtcd.KeyFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.KeyFile)) - } - case kubekeyv1alpha2.Kubeadm: - altNames := etcd.GenerateAltName(g.KubeConf, &runtime) - etcdCertSANs = append(etcdCertSANs, altNames.DNSNames...) - for _, ip := range altNames.IPs { - etcdCertSANs = append(etcdCertSANs, string(ip)) - } - } - - _, ApiServerArgs := util.GetArgs(templates.GetApiServerArgs(g.WithSecurityEnhancement, g.KubeConf.Cluster.Kubernetes.EnableAudit()), g.KubeConf.Cluster.Kubernetes.ApiServerArgs) - _, ControllerManagerArgs := util.GetArgs(templates.GetControllermanagerArgs(g.KubeConf.Cluster.Kubernetes.Version, g.WithSecurityEnhancement), g.KubeConf.Cluster.Kubernetes.ControllerManagerArgs) - _, SchedulerArgs := util.GetArgs(templates.GetSchedulerArgs(g.WithSecurityEnhancement), g.KubeConf.Cluster.Kubernetes.SchedulerArgs) - - checkCgroupDriver, err := templates.GetKubeletCgroupDriver(runtime, g.KubeConf) - if err != nil { - return err - } - - var ( - bootstrapToken, certificateKey string - // todo: if port needed - ) - if !g.IsInitConfiguration { - if v, ok := g.PipelineCache.Get(common.ClusterStatus); ok { - cluster := v.(*KubernetesStatus) - bootstrapToken = cluster.BootstrapToken - certificateKey = cluster.CertificateKey - } else { - return errors.New("get kubernetes cluster status by pipeline cache failed") - } - } - - templateAction := action.Template{ - Template: templates.KubeadmConfig, - Dst: filepath.Join(common.KubeConfigDir, templates.KubeadmConfig.Name()), - Data: util.Data{ - "IsInitCluster": g.IsInitConfiguration, - "ImageRepo": strings.TrimSuffix(images.GetImage(runtime, g.KubeConf, "kube-apiserver").ImageRepo(), "/kube-apiserver"), - "EtcdTypeIsKubeadm": g.KubeConf.Cluster.Etcd.Type == kubekeyv1alpha2.Kubeadm, - "EtcdCertSANs": etcdCertSANs, - "EtcdRepo": strings.TrimSuffix(images.GetImage(runtime, g.KubeConf, "etcd").ImageRepo(), "/etcd"), - "EtcdTag": images.GetImage(runtime, g.KubeConf, "etcd").Tag, - "CorednsRepo": strings.TrimSuffix(images.GetImage(runtime, g.KubeConf, "coredns").ImageRepo(), "/coredns"), - "CorednsTag": images.GetImage(runtime, g.KubeConf, "coredns").Tag, - "Version": g.KubeConf.Cluster.Kubernetes.Version, - "ClusterName": g.KubeConf.Cluster.Kubernetes.ClusterName, - "DNSDomain": g.KubeConf.Cluster.Kubernetes.DNSDomain, - "AdvertiseAddress": host.GetInternalAddress(), - "BindPort": kubekeyv1alpha2.DefaultApiserverPort, - "ControlPlaneEndpoint": fmt.Sprintf("%s:%d", g.KubeConf.Cluster.ControlPlaneEndpoint.Domain, g.KubeConf.Cluster.ControlPlaneEndpoint.Port), - "PodSubnet": g.KubeConf.Cluster.Network.KubePodsCIDR, - "ServiceSubnet": g.KubeConf.Cluster.Network.KubeServiceCIDR, - "CertSANs": g.KubeConf.Cluster.GenerateCertSANs(), - "ExternalEtcd": externalEtcd, - "NodeCidrMaskSize": g.KubeConf.Cluster.Kubernetes.NodeCidrMaskSize, - "CriSock": g.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint, - "ApiServerArgs": templates.UpdateFeatureGatesConfiguration(ApiServerArgs, g.KubeConf), - "EnableAudit": g.KubeConf.Cluster.Kubernetes.EnableAudit(), - "ControllerManagerArgs": templates.UpdateFeatureGatesConfiguration(ControllerManagerArgs, g.KubeConf), - "SchedulerArgs": templates.UpdateFeatureGatesConfiguration(SchedulerArgs, g.KubeConf), - "KubeletConfiguration": templates.GetKubeletConfiguration(runtime, g.KubeConf, g.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint, g.WithSecurityEnhancement), - "KubeProxyConfiguration": templates.GetKubeProxyConfiguration(g.KubeConf), - "IsV1beta3": versionutil.MustParseSemantic(g.KubeConf.Cluster.Kubernetes.Version).AtLeast(versionutil.MustParseSemantic("v1.22.0")), - "IsControlPlane": host.IsRole(common.Master), - "CgroupDriver": checkCgroupDriver, - "BootstrapToken": bootstrapToken, - "CertificateKey": certificateKey, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - } - return nil -} - -type KubeadmInit struct { - common.KubeAction -} - -func (k *KubeadmInit) Execute(runtime connector.Runtime) error { - initCmd := "/usr/local/bin/kubeadm init --config=/etc/kubernetes/kubeadm-config.yaml --ignore-preflight-errors=FileExisting-crictl,ImagePull" - - if k.KubeConf.Cluster.Kubernetes.DisableKubeProxy { - initCmd = initCmd + " --skip-phases=addon/kube-proxy" - } - - if _, err := runtime.GetRunner().SudoCmd(initCmd, true); err != nil { - // kubeadm reset and then retry - resetCmd := "/usr/local/bin/kubeadm reset -f" - if k.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint != "" { - resetCmd = resetCmd + " --cri-socket " + k.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint - } - _, _ = runtime.GetRunner().SudoCmd(resetCmd, true) - return errors.Wrap(errors.WithStack(err), "init kubernetes cluster failed") - } - return nil -} - -type CopyKubeConfigForControlPlane struct { - common.KubeAction -} - -func (c *CopyKubeConfigForControlPlane) Execute(runtime connector.Runtime) error { - createConfigDirCmd := "mkdir -p /root/.kube" - getKubeConfigCmd := "cp -f /etc/kubernetes/admin.conf /root/.kube/config" - cmd := strings.Join([]string{createConfigDirCmd, getKubeConfigCmd}, " && ") - if _, err := runtime.GetRunner().SudoCmd(cmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "copy kube config failed") - } - - userMkdir := "mkdir -p $HOME/.kube" - if _, err := runtime.GetRunner().Cmd(userMkdir, false); err != nil { - return errors.Wrap(errors.WithStack(err), "user mkdir $HOME/.kube failed") - } - - userCopyKubeConfig := "cp -f /etc/kubernetes/admin.conf $HOME/.kube/config" - if _, err := runtime.GetRunner().SudoCmd(userCopyKubeConfig, false); err != nil { - return errors.Wrap(errors.WithStack(err), "user copy /etc/kubernetes/admin.conf to $HOME/.kube/config failed") - } - - userId, err := runtime.GetRunner().Cmd("echo $(id -u)", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get user id failed") - } - - userGroupId, err := runtime.GetRunner().Cmd("echo $(id -g)", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get user group id failed") - } - - chownKubeConfig := fmt.Sprintf("chown -R %s:%s $HOME/.kube", userId, userGroupId) - if _, err := runtime.GetRunner().SudoCmd(chownKubeConfig, false); err != nil { - return errors.Wrap(errors.WithStack(err), "chown user kube config failed") - } - return nil -} - -type RemoveMasterTaint struct { - common.KubeAction -} - -func (r *RemoveMasterTaint) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "/usr/local/bin/kubectl taint nodes %s node-role.kubernetes.io/master=:NoSchedule-", - runtime.RemoteHost().GetName()), true); err != nil { - logger.Log.Warning(err.Error()) - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "/usr/local/bin/kubectl taint nodes %s node-role.kubernetes.io/control-plane=:NoSchedule-", - runtime.RemoteHost().GetName()), true); err != nil { - logger.Log.Warningf(err.Error()) - } - return nil -} - -type AddWorkerLabel struct { - common.KubeAction -} - -func (a *AddWorkerLabel) Execute(runtime connector.Runtime) error { - for _, host := range runtime.GetAllHosts() { - if host.IsRole(common.Worker) { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "/usr/local/bin/kubectl label --overwrite node %s node-role.kubernetes.io/worker=", - host.GetName()), true); err != nil { - return errors.Wrap(errors.WithStack(err), "add worker label failed") - } - } - } - - return nil -} - -type JoinNode struct { - common.KubeAction -} - -func (j *JoinNode) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubeadm join --config=/etc/kubernetes/kubeadm-config.yaml --ignore-preflight-errors=FileExisting-crictl,ImagePull", - true); err != nil { - resetCmd := "/usr/local/bin/kubeadm reset -f" - if j.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint != "" { - resetCmd = resetCmd + " --cri-socket " + j.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint - } - _, _ = runtime.GetRunner().SudoCmd(resetCmd, true) - return errors.Wrap(errors.WithStack(err), "join node failed") - } - return nil -} - -type KubeadmReset struct { - common.KubeAction -} - -func (k *KubeadmReset) Execute(runtime connector.Runtime) error { - resetCmd := "/usr/local/bin/kubeadm reset -f" - if k.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint != "" { - resetCmd = resetCmd + " --cri-socket " + k.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint - } - _, _ = runtime.GetRunner().SudoCmd(resetCmd, true) - return nil -} - -type FilterFirstMaster struct { - common.KubeAction -} - -func filterString(filter string, nodes []string) []string { - j := 0 - for _, v := range nodes { - if v != filter { - nodes[j] = v - j++ - } - } - resArr := nodes[:j] - return resArr -} - -func (f *FilterFirstMaster) Execute(runtime connector.Runtime) error { - firstMaster := runtime.GetHostsByRole(common.Master)[0].GetName() - //kubectl get node - var nodes []string - res, err := runtime.GetRunner().Cmd( - "sudo -E /usr/local/bin/kubectl get nodes | awk '{print $1}'", - true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "kubectl get nodes failed") - } - - if !strings.Contains(res, "\r\n") { - nodes = append(nodes, res) - } else { - nodes = strings.Split(res, "\r\n") - } - //nodes filter first master - resArr := filterString(firstMaster, nodes) - //nodes filter etcd nodes - for i := 0; i < len(runtime.GetHostsByRole(common.ETCD)); i++ { - etcdName := runtime.GetHostsByRole(common.ETCD)[i].GetName() - resArr = filterString(etcdName, resArr) - } - workerName := make(map[string]struct{}) - for j := 0; j < len(runtime.GetHostsByRole(common.Worker)); j++ { - workerName[runtime.GetHostsByRole(common.Worker)[j].GetName()] = struct{}{} - } - //make sure node is not the first master and etcd node name - var node string - for i := 0; i < len(resArr); i++ { - if _, ok := workerName[resArr[i]]; ok && resArr[i] == f.KubeConf.Arg.NodeName { - node = resArr[i] - break - } - } - - if node == "" { - return errors.New("" + - "1. check the node name in the config-sample.yaml\n" + - "2. check the node name in the Kubernetes cluster\n" + - "3. check the node name is the first master and etcd node name\n") - } - - f.PipelineCache.Set("dstNode", node) - return nil -} - -type FindNode struct { - common.KubeAction -} - -func (f *FindNode) Execute(runtime connector.Runtime) error { - var resArr []string - res, err := runtime.GetRunner().Cmd( - "sudo -E /usr/local/bin/kubectl get nodes | awk '$3 !~ /master|control-plane|ROLES/ {print $1}'", - true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "kubectl get nodes failed") - } - - if !strings.Contains(res, "\r\n") { - resArr = append(resArr, res) - } else { - resArr = strings.Split(res, "\r\n") - } - - workerName := make(map[string]struct{}) - for j := 0; j < len(runtime.GetHostsByRole(common.Worker)); j++ { - workerName[runtime.GetHostsByRole(common.Worker)[j].GetName()] = struct{}{} - } - - var node string - for i := 0; i < len(resArr); i++ { - if _, ok := workerName[resArr[i]]; ok && resArr[i] == f.KubeConf.Arg.NodeName { - node = resArr[i] - break - } - } - - if node == "" { - return errors.New("" + - "1. check the node name in the config-sample.yaml\n" + - "2. check the node name in the Kubernetes cluster\n" + - "3. only support to delete a worker\n") - } - - f.PipelineCache.Set("dstNode", node) - return nil -} - -type DrainNode struct { - common.KubeAction -} - -func (d *DrainNode) Execute(runtime connector.Runtime) error { - nodeName, ok := d.PipelineCache.Get("dstNode") - if !ok { - return errors.New("get dstNode failed by pipeline cache") - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "/usr/local/bin/kubectl drain %s --delete-emptydir-data --ignore-daemonsets --timeout=2m --force", nodeName), - true); err != nil { - return errors.Wrap(err, "drain the node failed") - } - return nil -} - -type KubectlDeleteNode struct { - common.KubeAction -} - -func (k *KubectlDeleteNode) Execute(runtime connector.Runtime) error { - nodeName, ok := k.PipelineCache.Get("dstNode") - if !ok { - return errors.New("get dstNode failed by pipeline cache") - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "/usr/local/bin/kubectl delete node %s", nodeName), - true); err != nil { - return errors.Wrap(err, "delete the node failed") - } - return nil -} - -type SetUpgradePlan struct { - common.KubeAction - Step UpgradeStep -} - -func (s *SetUpgradePlan) Execute(_ connector.Runtime) error { - currentVersion, ok := s.PipelineCache.GetMustString(common.K8sVersion) - if !ok { - return errors.New("get current Kubernetes version failed by pipeline cache") - } - - desiredVersion, ok := s.PipelineCache.GetMustString(common.DesiredK8sVersion) - if !ok { - return errors.New("get desired Kubernetes version failed by pipeline cache") - } - if cmp, err := versionutil.MustParseSemantic(currentVersion).Compare(desiredVersion); err != nil { - return err - } else if cmp == 1 { - logger.Log.Messagef( - common.LocalHost, - "The current version (%s) is greater than the target version (%s)", - currentVersion, desiredVersion) - os.Exit(0) - } - - if s.Step == ToV121 { - v122 := versionutil.MustParseSemantic("v1.22.0") - atLeast := versionutil.MustParseSemantic(desiredVersion).AtLeast(v122) - cmp, err := versionutil.MustParseSemantic(currentVersion).Compare("v1.21.5") - if err != nil { - return err - } - if atLeast && cmp <= 0 { - desiredVersion = "v1.21.5" - } - } - - s.PipelineCache.Set(common.PlanK8sVersion, desiredVersion) - return nil -} - -type CalculateNextVersion struct { - common.KubeAction -} - -func (c *CalculateNextVersion) Execute(_ connector.Runtime) error { - currentVersion, ok := c.PipelineCache.GetMustString(common.K8sVersion) - if !ok { - return errors.New("get current Kubernetes version failed by pipeline cache") - } - planVersion, ok := c.PipelineCache.GetMustString(common.PlanK8sVersion) - if !ok { - return errors.New("get upgrade plan Kubernetes version failed by pipeline cache") - } - nextVersionStr, err := calculateNextStr(currentVersion, planVersion) - if err != nil { - return errors.Wrap(err, "calculate next version failed") - } - c.KubeConf.Cluster.Kubernetes.Version = nextVersionStr - return nil -} - -func calculateNextStr(currentVersion, desiredVersion string) (string, error) { - current := versionutil.MustParseSemantic(currentVersion) - target := versionutil.MustParseSemantic(desiredVersion) - var nextVersionMinor uint - if target.Minor() == current.Minor() { - nextVersionMinor = current.Minor() - } else { - nextVersionMinor = current.Minor() + 1 - } - - if nextVersionMinor == target.Minor() { - if _, ok := files.FileSha256["kubeadm"]["amd64"][desiredVersion]; !ok { - return "", errors.Errorf("the target version %s is not supported", desiredVersion) - } - return desiredVersion, nil - } else { - nextVersionPatchList := make([]int, 0) - for supportVersionStr := range files.FileSha256["kubeadm"]["amd64"] { - supportVersion := versionutil.MustParseSemantic(supportVersionStr) - if supportVersion.Minor() == nextVersionMinor { - nextVersionPatchList = append(nextVersionPatchList, int(supportVersion.Patch())) - } - } - sort.Ints(nextVersionPatchList) - - nextVersion := current.WithMinor(nextVersionMinor) - if len(nextVersionPatchList) == 0 { - return "", errors.Errorf("Kubernetes minor version v%d.%d.x is not supported", nextVersion.Major(), nextVersion.Minor()) - } - nextVersion = nextVersion.WithPatch(uint(nextVersionPatchList[len(nextVersionPatchList)-1])) - - return fmt.Sprintf("v%s", nextVersion.String()), nil - } -} - -type RestartKubelet struct { - common.KubeAction - ModuleName string -} - -func (r *RestartKubelet) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - if _, err := runtime.GetRunner().SudoCmd("systemctl stop kubelet", false); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("stop kubelet failed: %s", host.GetName())) - } - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && systemctl restart kubelet", true); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("restart kubelet failed: %s", host.GetName())) - } - time.Sleep(10 * time.Second) - return nil -} - -type UpgradeKubeMaster struct { - common.KubeAction - ModuleName string -} - -func (u *UpgradeKubeMaster) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - - if err := KubeadmUpgradeTasks(runtime, u); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("upgrade cluster using kubeadm failed: %s", host.GetName())) - } - - if _, err := runtime.GetRunner().SudoCmd("systemctl stop kubelet", false); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("stop kubelet failed: %s", host.GetName())) - } - - if versionutil.MustParseSemantic(u.KubeConf.Cluster.Kubernetes.Version).AtLeast(versionutil.MustParseSemantic("v1.24.0")) { - if _, err := runtime.GetRunner().SudoCmd("sed -i 's/ --network-plugin=cni / /g' /var/lib/kubelet/kubeadm-flags.env", false); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("update kubelet config failed: %s", host.GetName())) - } - } - - if err := SetKubeletTasks(runtime, u.KubeAction); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("set kubelet failed: %s", host.GetName())) - } - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && systemctl restart kubelet", true); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("restart kubelet failed: %s", host.GetName())) - } - - time.Sleep(10 * time.Second) - return nil -} - -type UpgradeKubeWorker struct { - common.KubeAction - ModuleName string -} - -func (u *UpgradeKubeWorker) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubeadm upgrade node", true); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("upgrade node using kubeadm failed: %s", host.GetName())) - } - if _, err := runtime.GetRunner().SudoCmd("systemctl stop kubelet", true); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("stop kubelet failed: %s", host.GetName())) - } - if versionutil.MustParseSemantic(u.KubeConf.Cluster.Kubernetes.Version).AtLeast(versionutil.MustParseSemantic("v1.24.0")) { - if _, err := runtime.GetRunner().SudoCmd("sed -i 's/ --network-plugin=cni / /g' /var/lib/kubelet/kubeadm-flags.env", false); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("update kubelet config failed: %s", host.GetName())) - } - } - if err := SetKubeletTasks(runtime, u.KubeAction); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("set kubelet failed: %s", host.GetName())) - } - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && systemctl restart kubelet", true); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("restart kubelet failed: %s", host.GetName())) - } - time.Sleep(10 * time.Second) - return nil -} - -func KubeadmUpgradeTasks(runtime connector.Runtime, u *UpgradeKubeMaster) error { - host := runtime.RemoteHost() - - kubeadmUpgrade := &task.RemoteTask{ - Name: "KubeadmUpgrade", - Desc: "Upgrade cluster using kubeadm", - Hosts: []connector.Host{host}, - Prepare: new(NotEqualDesiredVersion), - Action: new(KubeadmUpgrade), - Parallel: false, - Retry: 3, - } - - copyKubeConfig := &task.RemoteTask{ - Name: "CopyKubeConfig", - Desc: "Copy admin.conf to ~/.kube/config", - Hosts: []connector.Host{host}, - Prepare: new(NotEqualDesiredVersion), - Action: new(CopyKubeConfigForControlPlane), - Parallel: false, - Retry: 2, - } - - tasks := []task.Interface{ - kubeadmUpgrade, - copyKubeConfig, - } - - for i := range tasks { - t := tasks[i] - t.Init(runtime, u.ModuleCache, u.PipelineCache) - if res := t.Execute(); res.IsFailed() { - return res.CombineErr() - } - } - return nil -} - -type KubeadmUpgrade struct { - common.KubeAction -} - -func (k *KubeadmUpgrade) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - fmt.Println(k.KubeConf.Cluster.Kubernetes.Version) - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "timeout -k 600s 600s /usr/local/bin/kubeadm upgrade apply %s -y "+ - "--ignore-preflight-errors=all "+ - "--allow-experimental-upgrades "+ - "--allow-release-candidate-upgrades "+ - "--etcd-upgrade=false "+ - "--certificate-renewal=true ", - k.KubeConf.Cluster.Kubernetes.Version), false); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("upgrade master failed: %s", host.GetName())) - } - return nil -} - -func SetKubeletTasks(runtime connector.Runtime, kubeAction common.KubeAction) error { - host := runtime.RemoteHost() - chmodKubelet := &task.RemoteTask{ - Name: "ChownKubelet", - Desc: "Change kubelet owner", - Hosts: []connector.Host{host}, - Prepare: new(NotEqualDesiredVersion), - Action: new(ChmodKubelet), - Parallel: false, - Retry: 2, - } - - enableKubelet := &task.RemoteTask{ - Name: "EnableKubelet", - Desc: "enable kubelet service", - Hosts: []connector.Host{host}, - Prepare: new(NotEqualDesiredVersion), - Action: new(EnableKubelet), - Parallel: false, - Retry: 5, - } - - tasks := []task.Interface{ - chmodKubelet, - enableKubelet, - } - - for i := range tasks { - t := tasks[i] - t.Init(runtime, kubeAction.ModuleCache, kubeAction.PipelineCache) - if res := t.Execute(); res.IsFailed() { - return res.CombineErr() - } - } - return nil -} - -//type UpgradeDNS struct { -// common.KubeAction -// ModuleName string -//} -// -//func (r *UpgradeDNS) Execute(runtime connector.Runtime) error { -// if err := UpgradeCoredns(runtime, r.KubeAction); err != nil { -// return errors.Wrap(errors.WithStack(err), "re-config coredns failed") -// } -// return nil -//} -// -//func UpgradeCoredns(runtime connector.Runtime, kubeAction common.KubeAction) error { -// host := runtime.RemoteHost() -// -// generateCoreDNSSvc := &task.RemoteTask{ -// Name: "GenerateCoreDNS", -// Desc: "generate coredns manifests", -// Hosts: []connector.Host{host}, -// Prepare: &prepare.PrepareCollection{ -// new(common.OnlyFirstMaster), -// }, -// Action: &action.Template{ -// Template: dnsTemplates.Coredns, -// Dst: filepath.Join(common.KubeConfigDir, dnsTemplates.Coredns.Name()), -// Data: util.Data{ -// "ClusterIP": kubeAction.KubeConf.Cluster.CorednsClusterIP(), -// "CorednsImage": images.GetImage(runtime, kubeAction.KubeConf, "coredns").ImageName(), -// "DNSEtchHsts": kubeAction.KubeConf.Cluster.DNS.DNSEtcHosts, -// }, -// }, -// Parallel: true, -// } -// -// override := &task.RemoteTask{ -// Name: "UpgradeCoreDNS", -// Desc: "upgrade coredns", -// Hosts: []connector.Host{host}, -// Prepare: &prepare.PrepareCollection{ -// new(common.OnlyFirstMaster), -// }, -// Action: new(dns.DeployCoreDNS), -// Parallel: false, -// } -// -// generateNodeLocalDNS := &task.RemoteTask{ -// Name: "GenerateNodeLocalDNS", -// Desc: "generate nodelocaldns", -// Hosts: []connector.Host{host}, -// Prepare: &prepare.PrepareCollection{ -// new(common.OnlyFirstMaster), -// new(dns.EnableNodeLocalDNS), -// }, -// Action: &action.Template{ -// Template: dnsTemplates.NodeLocalDNSService, -// Dst: filepath.Join(common.KubeConfigDir, dnsTemplates.NodeLocalDNSService.Name()), -// Data: util.Data{ -// "NodelocaldnsImage": images.GetImage(runtime, kubeAction.KubeConf, "k8s-dns-node-cache").ImageName(), -// }, -// }, -// Parallel: true, -// } -// -// applyNodeLocalDNS := &task.RemoteTask{ -// Name: "DeployNodeLocalDNS", -// Desc: "deploy nodelocaldns", -// Hosts: []connector.Host{host}, -// Prepare: &prepare.PrepareCollection{ -// new(common.OnlyFirstMaster), -// new(dns.EnableNodeLocalDNS), -// }, -// Action: new(dns.DeployNodeLocalDNS), -// Parallel: true, -// Retry: 5, -// } -// -// generateNodeLocalDNSConfigMap := &task.RemoteTask{ -// Name: "GenerateNodeLocalDNSConfigMap", -// Desc: "generate nodelocaldns configmap", -// Hosts: []connector.Host{host}, -// Prepare: &prepare.PrepareCollection{ -// new(common.OnlyFirstMaster), -// new(dns.EnableNodeLocalDNS), -// new(dns.NodeLocalDNSConfigMapNotExist), -// }, -// Action: new(dns.GenerateNodeLocalDNSConfigMap), -// Parallel: true, -// } -// -// applyNodeLocalDNSConfigMap := &task.RemoteTask{ -// Name: "ApplyNodeLocalDNSConfigMap", -// Desc: "apply nodelocaldns configmap", -// Hosts: []connector.Host{host}, -// Prepare: &prepare.PrepareCollection{ -// new(common.OnlyFirstMaster), -// new(dns.EnableNodeLocalDNS), -// new(dns.NodeLocalDNSConfigMapNotExist), -// }, -// Action: new(dns.ApplyNodeLocalDNSConfigMap), -// Parallel: true, -// Retry: 5, -// } -// -// tasks := []task.Interface{ -// override, -// generateCoreDNSSvc, -// override, -// generateNodeLocalDNS, -// applyNodeLocalDNS, -// generateNodeLocalDNSConfigMap, -// applyNodeLocalDNSConfigMap, -// } -// -// for i := range tasks { -// t := tasks[i] -// t.Init(runtime, kubeAction.ModuleCache, kubeAction.PipelineCache) -// if res := t.Execute(); res.IsFailed() { -// return res.CombineErr() -// } -// } -// return nil -//} - -type SetCurrentK8sVersion struct { - common.KubeAction -} - -func (s *SetCurrentK8sVersion) Execute(_ connector.Runtime) error { - s.PipelineCache.Set(common.K8sVersion, s.KubeConf.Cluster.Kubernetes.Version) - return nil -} - -type SaveKubeConfig struct { - common.KubeAction -} - -func (s *SaveKubeConfig) Execute(runtime connector.Runtime) error { - status, ok := s.PipelineCache.Get(common.ClusterStatus) - if !ok { - return errors.New("get kubernetes status failed by pipeline cache") - } - cluster := status.(*KubernetesStatus) - kubeConfigStr := cluster.KubeConfig - - clusterPublicAddress := s.KubeConf.Cluster.ControlPlaneEndpoint.Address - master1 := runtime.GetHostsByRole(common.Master)[0] - if clusterPublicAddress == master1.GetInternalAddress() || clusterPublicAddress == "" { - clusterPublicAddress = master1.GetAddress() - } - - oldServer := fmt.Sprintf("https://%s:%d", s.KubeConf.Cluster.ControlPlaneEndpoint.Domain, s.KubeConf.Cluster.ControlPlaneEndpoint.Port) - newServer := fmt.Sprintf("https://%s:%d", clusterPublicAddress, s.KubeConf.Cluster.ControlPlaneEndpoint.Port) - newKubeConfigStr := strings.Replace(kubeConfigStr, oldServer, newServer, -1) - kubeConfigBase64 := base64.StdEncoding.EncodeToString([]byte(newKubeConfigStr)) - - config, err := clientcmd.NewClientConfigFromBytes([]byte(newKubeConfigStr)) - if err != nil { - return err - } - restConfig, err := config.ClientConfig() - if err != nil { - return err - } - clientsetForCluster, err := kube.NewForConfig(restConfig) - if err != nil { - return err - } - - namespace := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kubekey-system", - }, - } - if _, err := clientsetForCluster. - CoreV1(). - Namespaces(). - Get(context.TODO(), namespace.Name, metav1.GetOptions{}); kubeerrors.IsNotFound(err) { - if _, err := clientsetForCluster. - CoreV1(). - Namespaces(). - Create(context.TODO(), namespace, metav1.CreateOptions{}); err != nil { - return err - } - } else { - return err - } - - cm := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-kubeconfig", s.KubeConf.ClusterName), - }, - Data: map[string]string{ - "kubeconfig": kubeConfigBase64, - }, - } - - if _, err := clientsetForCluster. - CoreV1(). - ConfigMaps("kubekey-system"). - Get(context.TODO(), cm.Name, metav1.GetOptions{}); kubeerrors.IsNotFound(err) { - if _, err := clientsetForCluster. - CoreV1(). - ConfigMaps("kubekey-system"). - Create(context.TODO(), cm, metav1.CreateOptions{}); err != nil { - return err - } - } else { - if _, err := clientsetForCluster. - CoreV1(). - ConfigMaps("kubekey-system"). - Update(context.TODO(), cm, metav1.UpdateOptions{}); err != nil { - return err - } - } - return nil -} - -type ConfigureKubernetes struct { - common.KubeAction -} - -func (c *ConfigureKubernetes) Execute(runtime connector.Runtime) error { - hosts := runtime.GetHostsByRole(common.K8s) - - for j := 0; j < len(hosts); j++ { - kubeHost := hosts[j].(*kubekeyv1alpha2.KubeHost) - for k, v := range kubeHost.Labels { - labelCmd := fmt.Sprintf("/usr/local/bin/kubectl label --overwrite node %s %s=%s", hosts[j].GetName(), k, v) - _, err := runtime.GetRunner().SudoCmd(labelCmd, true) - if err != nil { - return err - } - } - } - return nil -} - -type EtcdSecurityEnhancemenAction struct { - common.KubeAction - ModuleName string -} - -func (s *EtcdSecurityEnhancemenAction) Execute(runtime connector.Runtime) error { - chmodEtcdCertsDirCmd := "chmod 700 /etc/ssl/etcd/ssl" - chmodEtcdCertsCmd := "chmod 600 /etc/ssl/etcd/ssl/*" - chmodEtcdDataDirCmd := "chmod 700 /var/lib/etcd" - chmodEtcdCmd := "chmod 550 /usr/local/bin/etcd*" - - chownEtcdCertsDirCmd := "chown root:root /etc/ssl/etcd/ssl" - chownEtcdCertsCmd := "chown root:root /etc/ssl/etcd/ssl/*" - chownEtcdDataDirCmd := "chown etcd:etcd /var/lib/etcd" - chownEtcdCmd := "chown root:root /usr/local/bin/etcd*" - - ETCDcmds := []string{chmodEtcdCertsDirCmd, chmodEtcdCertsCmd, chmodEtcdDataDirCmd, chmodEtcdCmd, chownEtcdCertsDirCmd, chownEtcdCertsCmd, chownEtcdDataDirCmd, chownEtcdCmd} - - if _, err := runtime.GetRunner().SudoCmd(strings.Join(ETCDcmds, " && "), true); err != nil { - return errors.Wrap(errors.WithStack(err), "Updating permissions failed.") - } - - return nil -} - -type MasterSecurityEnhancemenAction struct { - common.KubeAction - ModuleName string -} - -func (k *MasterSecurityEnhancemenAction) Execute(runtime connector.Runtime) error { - // Control-plane Security Enhancemen - chmodKubernetesDirCmd := "chmod 644 /etc/kubernetes" - chownKubernetesDirCmd := "chown root:root /etc/kubernetes" - - chmodKubernetesConfigCmd := "chmod 600 -R /etc/kubernetes" - chownKubernetesConfigCmd := "chown root:root -R /etc/kubernetes/*" - - chmodKubenretesManifestsDirCmd := "chmod 644 /etc/kubernetes/manifests" - chownKubenretesManifestsDirCmd := "chown root:root /etc/kubernetes/manifests" - - chmodKubenretesCertsDirCmd := "chmod 644 /etc/kubernetes/pki" - chownKubenretesCertsDirCmd := "chown root:root /etc/kubernetes/pki" - - // node Security Enhancemen - chmodCniConfigDir := "chmod 600 -R /etc/cni/net.d" - chownCniConfigDir := "chown root:root -R /etc/cni/net.d" - - chmodBinDir := "chmod 550 /usr/local/bin/" - chownBinDir := "chown root:root /usr/local/bin/" - - chmodKubeCmd := "chmod 550 -R /usr/local/bin/kube*" - chownKubeCmd := "chown root:root -R /usr/local/bin/kube*" - - chmodHelmCmd := "chmod 550 /usr/local/bin/helm" - chownHelmCmd := "chown root:root /usr/local/bin/helm" - - chmodCniDir := "chmod 550 -R /opt/cni/bin" - chownCniDir := "chown root:root -R /opt/cni/bin" - - chmodKubeletConfig := "chmod 640 /var/lib/kubelet/config.yaml && chmod 640 -R /etc/systemd/system/kubelet.service*" - chownKubeletConfig := "chown root:root /var/lib/kubelet/config.yaml && chown root:root -R /etc/systemd/system/kubelet.service*" - - chmodCertsRenew := "chmod 640 /etc/systemd/system/k8s-certs-renew*" - chownCertsRenew := "chown root:root /etc/systemd/system/k8s-certs-renew*" - - chmodMasterCmds := []string{chmodKubernetesConfigCmd, chmodKubernetesDirCmd, chmodKubenretesManifestsDirCmd, chmodKubenretesCertsDirCmd} - if _, err := runtime.GetRunner().SudoCmd(strings.Join(chmodMasterCmds, " && "), true); err != nil { - return errors.Wrap(errors.WithStack(err), "Updating permissions failed.") - } - chownMasterCmds := []string{chownKubernetesConfigCmd, chownKubernetesDirCmd, chownKubenretesManifestsDirCmd, chownKubenretesCertsDirCmd} - if _, err := runtime.GetRunner().SudoCmd(strings.Join(chownMasterCmds, " && "), true); err != nil { - return errors.Wrap(errors.WithStack(err), "Updating permissions failed.") - } - - chmodNodesCmds := []string{chmodBinDir, chmodKubeCmd, chmodHelmCmd, chmodCniDir, chmodCniConfigDir, chmodKubeletConfig, chmodCertsRenew} - if _, err := runtime.GetRunner().SudoCmd(strings.Join(chmodNodesCmds, " && "), true); err != nil { - return errors.Wrap(errors.WithStack(err), "Updating permissions failed.") - } - chownNodesCmds := []string{chownBinDir, chownKubeCmd, chownHelmCmd, chownCniDir, chownCniConfigDir, chownKubeletConfig, chownCertsRenew} - if _, err := runtime.GetRunner().SudoCmd(strings.Join(chownNodesCmds, " && "), true); err != nil { - return errors.Wrap(errors.WithStack(err), "Updating permissions failed.") - } - - return nil -} - -type NodesSecurityEnhancemenAction struct { - common.KubeAction - ModuleName string -} - -func (n *NodesSecurityEnhancemenAction) Execute(runtime connector.Runtime) error { - // Control-plane Security Enhancemen - chmodKubernetesDirCmd := "chmod 644 /etc/kubernetes" - chownKubernetesDirCmd := "chown root:root /etc/kubernetes" - - chmodKubernetesConfigCmd := "chmod 600 -R /etc/kubernetes" - chownKubernetesConfigCmd := "chown root:root -R /etc/kubernetes" - - chmodKubenretesManifestsDirCmd := "chmod 644 /etc/kubernetes/manifests" - chownKubenretesManifestsDirCmd := "chown root:root /etc/kubernetes/manifests" - - chmodKubenretesCertsDirCmd := "chmod 644 /etc/kubernetes/pki" - chownKubenretesCertsDirCmd := "chown root:root /etc/kubernetes/pki" - - // node Security Enhancemen - chmodCniConfigDir := "chmod 600 -R /etc/cni/net.d" - chownCniConfigDir := "chown root:root -R /etc/cni/net.d" - - chmodBinDir := "chmod 550 /usr/local/bin/" - chownBinDir := "chown root:root /usr/local/bin/" - - chmodKubeCmd := "chmod 550 -R /usr/local/bin/kube*" - chownKubeCmd := "chown root:root -R /usr/local/bin/kube*" - - chmodHelmCmd := "chmod 550 /usr/local/bin/helm" - chownHelmCmd := "chown root:root /usr/local/bin/helm" - - chmodCniDir := "chmod 550 -R /opt/cni/bin" - chownCniDir := "chown root:root -R /opt/cni/bin" - - chmodKubeletConfig := "chmod 640 /var/lib/kubelet/config.yaml && chmod 640 -R /etc/systemd/system/kubelet.service*" - chownKubeletConfig := "chown root:root /var/lib/kubelet/config.yaml && chown root:root -R /etc/systemd/system/kubelet.service*" - - chmodMasterCmds := []string{chmodKubernetesConfigCmd, chmodKubernetesDirCmd, chmodKubenretesManifestsDirCmd, chmodKubenretesCertsDirCmd} - if _, err := runtime.GetRunner().SudoCmd(strings.Join(chmodMasterCmds, " && "), true); err != nil { - return errors.Wrap(errors.WithStack(err), "Updating permissions failed.") - } - chownMasterCmds := []string{chownKubernetesConfigCmd, chownKubernetesDirCmd, chownKubenretesManifestsDirCmd, chownKubenretesCertsDirCmd} - if _, err := runtime.GetRunner().SudoCmd(strings.Join(chownMasterCmds, " && "), true); err != nil { - return errors.Wrap(errors.WithStack(err), "Updating permissions failed.") - } - - chmodNodesCmds := []string{chmodBinDir, chmodKubeCmd, chmodHelmCmd, chmodCniDir, chmodCniConfigDir, chmodKubeletConfig} - if _, err := runtime.GetRunner().SudoCmd(strings.Join(chmodNodesCmds, " && "), true); err != nil { - return errors.Wrap(errors.WithStack(err), "Updating permissions failed.") - } - chownNodesCmds := []string{chownBinDir, chownKubeCmd, chownHelmCmd, chownCniDir, chownCniConfigDir, chownKubeletConfig} - if _, err := runtime.GetRunner().SudoCmd(strings.Join(chownNodesCmds, " && "), true); err != nil { - return errors.Wrap(errors.WithStack(err), "Updating permissions failed.") - } - - return nil -} diff --git a/cmd/kk/pkg/kubernetes/templates/audit.go b/cmd/kk/pkg/kubernetes/templates/audit.go deleted file mode 100644 index 83b19e023..000000000 --- a/cmd/kk/pkg/kubernetes/templates/audit.go +++ /dev/null @@ -1,168 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -// AuditPolicy defines the template of kube-apiserver audit-policy. -var AuditPolicy = template.Must(template.New("audit-policy.yaml").Parse( - dedent.Dedent(`apiVersion: audit.k8s.io/v1 -kind: Policy -rules: - # The following requests were manually identified as high-volume and low-risk, - # so drop them. - - level: None - users: ["system:kube-proxy"] - verbs: ["watch"] - resources: - - group: "" # core - resources: ["endpoints", "services", "services/status"] - - level: None - users: ["system:unsecured"] - namespaces: ["kube-system"] - verbs: ["get"] - resources: - - group: "" # core - resources: ["configmaps"] - - level: None - users: ["kubelet"] # legacy kubelet identity - verbs: ["get"] - resources: - - group: "" # core - resources: ["nodes", "nodes/status"] - - level: None - userGroups: ["system:nodes"] - verbs: ["get"] - resources: - - group: "" # core - resources: ["nodes", "nodes/status"] - - level: None - users: - - system:kube-controller-manager - - system:kube-scheduler - - system:serviceaccount:kube-system:endpoint-controller - verbs: ["get", "update"] - namespaces: ["kube-system"] - resources: - - group: "" # core - resources: ["endpoints"] - - level: None - users: ["system:apiserver"] - verbs: ["get"] - resources: - - group: "" # core - resources: ["namespaces", "namespaces/status", "namespaces/finalize"] - # Don't log HPA fetching metrics. - - level: None - users: - - system:kube-controller-manager - verbs: ["get", "list"] - resources: - - group: "metrics.k8s.io" - # Don't log these read-only URLs. - - level: None - nonResourceURLs: - - /healthz* - - /version - - /swagger* - # Don't log events requests. - - level: None - resources: - - group: "" # core - resources: ["events"] - # Secrets, ConfigMaps, TokenRequest and TokenReviews can contain sensitive & binary data, - # so only log at the Metadata level. - - level: Metadata - resources: - - group: "" # core - resources: ["secrets", "configmaps", "serviceaccounts/token"] - - group: authentication.k8s.io - resources: ["tokenreviews"] - omitStages: - - "RequestReceived" - # Get responses can be large; skip them. - - level: Request - verbs: ["get", "list", "watch"] - resources: - - group: "" # core - - group: "admissionregistration.k8s.io" - - group: "apiextensions.k8s.io" - - group: "apiregistration.k8s.io" - - group: "apps" - - group: "authentication.k8s.io" - - group: "authorization.k8s.io" - - group: "autoscaling" - - group: "batch" - - group: "certificates.k8s.io" - - group: "extensions" - - group: "metrics.k8s.io" - - group: "networking.k8s.io" - - group: "policy" - - group: "rbac.authorization.k8s.io" - - group: "settings.k8s.io" - - group: "storage.k8s.io" - omitStages: - - "RequestReceived" - # Default level for known APIs - - level: RequestResponse - resources: - - group: "" # core - - group: "admissionregistration.k8s.io" - - group: "apiextensions.k8s.io" - - group: "apiregistration.k8s.io" - - group: "apps" - - group: "authentication.k8s.io" - - group: "authorization.k8s.io" - - group: "autoscaling" - - group: "batch" - - group: "certificates.k8s.io" - - group: "extensions" - - group: "metrics.k8s.io" - - group: "networking.k8s.io" - - group: "policy" - - group: "rbac.authorization.k8s.io" - - group: "settings.k8s.io" - - group: "storage.k8s.io" - omitStages: - - "RequestReceived" - # Default level for all other requests. - - level: Metadata - omitStages: - - "RequestReceived" - `))) - -// AuditWebhook defines the template of kube-apiserver audit-webhook. -var AuditWebhook = template.Must(template.New("audit-webhook.yaml").Parse( - dedent.Dedent(`apiVersion: v1 -kind: Config -clusters: -- name: kube-auditing - cluster: - server: https://SHOULD_BE_REPLACED:6443/audit/webhook/event - insecure-skip-tls-verify: true -contexts: -- context: - cluster: kube-auditing - user: "" - name: default-context -current-context: default-context -preferences: {} -users: [] - `))) diff --git a/cmd/kk/pkg/kubernetes/templates/kubeadm_config.go b/cmd/kk/pkg/kubernetes/templates/kubeadm_config.go deleted file mode 100644 index 17d75a0a5..000000000 --- a/cmd/kk/pkg/kubernetes/templates/kubeadm_config.go +++ /dev/null @@ -1,471 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "fmt" - "strings" - "text/template" - - "github.com/lithammer/dedent" - "github.com/pkg/errors" - "gopkg.in/yaml.v3" - versionutil "k8s.io/apimachinery/pkg/util/version" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" -) - -var ( - // KubeadmConfig defines the template of kubeadm configuration file. - KubeadmConfig = template.Must(template.New("kubeadm-config.yaml").Funcs(utils.FuncMap).Parse( - dedent.Dedent(` -{{- if .IsInitCluster -}} ---- -apiVersion: kubeadm.k8s.io/{{ if .IsV1beta3 }}v1beta3{{ else }}v1beta2{{ end }} -kind: ClusterConfiguration -etcd: -{{- if .EtcdTypeIsKubeadm }} - local: - imageRepository: {{ .EtcdRepo }} - imageTag: {{ .EtcdTag }} - serverCertSANs: - {{- range .ExternalEtcd.Endpoints }} - - {{ . }} - {{- end }} -{{- else }} - external: - endpoints: - {{- range .ExternalEtcd.Endpoints }} - - {{ . }} - {{- end }} -{{- if .ExternalEtcd.CAFile }} - caFile: {{ .ExternalEtcd.CAFile }} -{{- end }} -{{- if .ExternalEtcd.CertFile }} - certFile: {{ .ExternalEtcd.CertFile }} -{{- end }} -{{- if .ExternalEtcd.KeyFile }} - keyFile: {{ .ExternalEtcd.KeyFile }} -{{- end }} -{{- end }} -dns: - type: CoreDNS - imageRepository: {{ .CorednsRepo }} - imageTag: {{ .CorednsTag }} -imageRepository: {{ .ImageRepo }} -kubernetesVersion: {{ .Version }} -certificatesDir: /etc/kubernetes/pki -clusterName: {{ .ClusterName }} -controlPlaneEndpoint: {{ .ControlPlaneEndpoint }} -networking: - dnsDomain: {{ .DNSDomain }} - podSubnet: {{ .PodSubnet }} - serviceSubnet: {{ .ServiceSubnet }} -apiServer: - extraArgs: -{{ toYaml .ApiServerArgs | indent 4}} - certSANs: - {{- range .CertSANs }} - - "{{ . }}" - {{- end }} -{{- if .EnableAudit }} - extraVolumes: - - name: k8s-audit - hostPath: /etc/kubernetes/audit - mountPath: /etc/kubernetes/audit - pathType: DirectoryOrCreate -{{- end }} -controllerManager: - extraArgs: - node-cidr-mask-size: "{{ .NodeCidrMaskSize }}" -{{ toYaml .ControllerManagerArgs | indent 4 }} - extraVolumes: - - name: host-time - hostPath: /etc/localtime - mountPath: /etc/localtime - readOnly: true -scheduler: - extraArgs: -{{ toYaml .SchedulerArgs | indent 4 }} - ---- -apiVersion: kubeadm.k8s.io/{{ if .IsV1beta3 }}v1beta3{{ else }}v1beta2{{ end }} -kind: InitConfiguration -localAPIEndpoint: - advertiseAddress: {{ .AdvertiseAddress }} - bindPort: {{ .BindPort }} -nodeRegistration: -{{- if .CriSock }} - criSocket: {{ .CriSock }} -{{- end }} - kubeletExtraArgs: - cgroup-driver: {{ .CgroupDriver }} ---- -apiVersion: kubeproxy.config.k8s.io/v1alpha1 -kind: KubeProxyConfiguration -{{ toYaml .KubeProxyConfiguration }} ---- -apiVersion: kubelet.config.k8s.io/v1beta1 -kind: KubeletConfiguration -{{ toYaml .KubeletConfiguration }} - -{{- else -}} ---- -apiVersion: kubeadm.k8s.io/{{ if .IsV1beta3 }}v1beta3{{ else }}v1beta2{{ end }} -kind: JoinConfiguration -discovery: - bootstrapToken: - apiServerEndpoint: {{ .ControlPlaneEndpoint }} - token: "{{ .BootstrapToken }}" - unsafeSkipCAVerification: true - tlsBootstrapToken: "{{ .BootstrapToken }}" -{{- if .IsControlPlane }} -controlPlane: - localAPIEndpoint: - advertiseAddress: {{ .AdvertiseAddress }} - bindPort: {{ .BindPort }} - certificateKey: {{ .CertificateKey }} -{{- end }} -nodeRegistration: -{{- if .CriSock }} - criSocket: {{ .CriSock }} -{{- end }} - kubeletExtraArgs: - cgroup-driver: {{ .CgroupDriver }} - -{{- end }} - `))) -) - -var ( - // ref: https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/ - FeatureGatesDefaultConfiguration = map[string]bool{ - "RotateKubeletServerCertificate": true, //k8s 1.7+ - "TTLAfterFinished": true, //k8s 1.12+ - } - FeatureGatesSecurityDefaultConfiguration = map[string]bool{ - "RotateKubeletServerCertificate": true, //k8s 1.7+ - "TTLAfterFinished": true, //k8s 1.12+ - "SeccompDefault": true, //kubelet - } - - ApiServerArgs = map[string]string{ - "bind-address": "0.0.0.0", - } - ApiServerSecurityArgs = map[string]string{ - "bind-address": "0.0.0.0", - "authorization-mode": "Node,RBAC", - // --enable-admission-plugins=EventRateLimit must have a configuration file - "enable-admission-plugins": "AlwaysPullImages,ServiceAccount,NamespaceLifecycle,NodeRestriction,LimitRanger,ResourceQuota,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,PodNodeSelector,PodSecurity", - // "audit-log-path": "/var/log/apiserver/audit.log", // need audit policy - "profiling": "false", - "request-timeout": "120s", - "service-account-lookup": "true", - "tls-min-version": "VersionTLS12", - "tls-cipher-suites": "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", - } - auditArgs = map[string]string{ - "audit-log-format": "json", - "audit-log-maxbackup": "2", - "audit-log-maxsize": "200", - "audit-policy-file": "/etc/kubernetes/audit/audit-policy.yaml", - "audit-webhook-config-file": "/etc/kubernetes/audit/audit-webhook.yaml", - } - ControllermanagerArgs = map[string]string{ - "bind-address": "0.0.0.0", - "cluster-signing-duration": "87600h", - } - ControllermanagerSecurityArgs = map[string]string{ - "bind-address": "127.0.0.1", - "cluster-signing-duration": "87600h", - "profiling": "false", - "terminated-pod-gc-threshold": "50", - "use-service-account-credentials": "true", - } - SchedulerArgs = map[string]string{ - "bind-address": "0.0.0.0", - } - SchedulerSecurityArgs = map[string]string{ - "bind-address": "127.0.0.1", - "profiling": "false", - } -) - -func GetApiServerArgs(securityEnhancement bool, enableAudit bool) map[string]string { - if securityEnhancement { - if enableAudit { - for k, v := range auditArgs { - ApiServerSecurityArgs[k] = v - } - } - return ApiServerSecurityArgs - } - - if enableAudit { - for k, v := range auditArgs { - ApiServerArgs[k] = v - } - } - - return ApiServerArgs -} - -func GetControllermanagerArgs(version string, securityEnhancement bool) map[string]string { - var args map[string]string - if securityEnhancement { - args = copyStringMap(ControllermanagerSecurityArgs) - } else { - args = copyStringMap(ControllermanagerArgs) - } - - if versionutil.MustParseSemantic(version).LessThan(versionutil.MustParseSemantic("1.19.0")) { - delete(args, "cluster-signing-duration") - args["experimental-cluster-signing-duration"] = "87600h" - } - return args -} - -func GetSchedulerArgs(securityEnhancement bool) map[string]string { - if securityEnhancement { - return SchedulerSecurityArgs - } - return SchedulerArgs -} - -func UpdateFeatureGatesConfiguration(args map[string]string, kubeConf *common.KubeConf) map[string]string { - var featureGates []string - - for k, v := range kubeConf.Cluster.Kubernetes.FeatureGates { - featureGates = append(featureGates, fmt.Sprintf("%s=%v", k, v)) - } - - for k, v := range FeatureGatesDefaultConfiguration { - // When kubernetes version is less than 1.21,`CSIStorageCapacity` should not be set. - if k == "CSIStorageCapacity" && - versionutil.MustParseSemantic(kubeConf.Cluster.Kubernetes.Version).LessThan(versionutil.MustParseSemantic("v1.21.0")) { - continue - } - if k == "TTLAfterFinished" && - versionutil.MustParseSemantic(kubeConf.Cluster.Kubernetes.Version).AtLeast(versionutil.MustParseSemantic("v1.24.0")) { - continue - } - - if _, ok := kubeConf.Cluster.Kubernetes.FeatureGates[k]; !ok { - featureGates = append(featureGates, fmt.Sprintf("%s=%v", k, v)) - } - } - - args["feature-gates"] = strings.Join(featureGates, ",") - - return args -} - -func GetKubeletConfiguration(runtime connector.Runtime, kubeConf *common.KubeConf, criSock string, securityEnhancement bool) map[string]interface{} { - // When kubernetes version is less than 1.21,`CSIStorageCapacity` should not be set. - cmp, _ := versionutil.MustParseSemantic(kubeConf.Cluster.Kubernetes.Version).Compare("v1.21.0") - if cmp == -1 { - delete(FeatureGatesDefaultConfiguration, "CSIStorageCapacity") - } - - defaultKubeletConfiguration := map[string]interface{}{ - "clusterDomain": kubeConf.Cluster.Kubernetes.DNSDomain, - "clusterDNS": []string{kubeConf.Cluster.ClusterDNS()}, - "maxPods": kubeConf.Cluster.Kubernetes.MaxPods, - "podPidsLimit": kubeConf.Cluster.Kubernetes.PodPidsLimit, - "rotateCertificates": true, - "kubeReserved": map[string]string{ - "cpu": "200m", - "memory": "250Mi", - }, - "systemReserved": map[string]string{ - "cpu": "200m", - "memory": "250Mi", - }, - "evictionHard": map[string]string{ - "memory.available": "5%", - "pid.available": "10%", - }, - "evictionSoft": map[string]string{ - "memory.available": "10%", - }, - "evictionSoftGracePeriod": map[string]string{ - "memory.available": "2m", - }, - "evictionMaxPodGracePeriod": 120, - "evictionPressureTransitionPeriod": "30s", - "featureGates": FeatureGatesDefaultConfiguration, - } - - if securityEnhancement { - defaultKubeletConfiguration["readOnlyPort"] = 0 - defaultKubeletConfiguration["protectKernelDefaults"] = true - defaultKubeletConfiguration["eventRecordQPS"] = 1 - defaultKubeletConfiguration["streamingConnectionIdleTimeout"] = "5m" - defaultKubeletConfiguration["makeIPTablesUtilChains"] = true - defaultKubeletConfiguration["tlsCipherSuites"] = []string{ - "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", - "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", - } - defaultKubeletConfiguration["featureGates"] = FeatureGatesSecurityDefaultConfiguration - } - - cgroupDriver, err := GetKubeletCgroupDriver(runtime, kubeConf) - if err != nil { - logger.Log.Fatal(err) - } - if len(cgroupDriver) == 0 { - defaultKubeletConfiguration["cgroupDriver"] = "systemd" - } - - if len(criSock) != 0 { - defaultKubeletConfiguration["containerLogMaxSize"] = "5Mi" - defaultKubeletConfiguration["containerLogMaxFiles"] = 3 - } - - customKubeletConfiguration := make(map[string]interface{}) - if len(kubeConf.Cluster.Kubernetes.KubeletConfiguration.Raw) != 0 { - err := yaml.Unmarshal(kubeConf.Cluster.Kubernetes.KubeletConfiguration.Raw, &customKubeletConfiguration) - if err != nil { - logger.Log.Fatal("failed to parse kubelet configuration") - } - } - - kubeletConfiguration := make(map[string]interface{}) - if len(customKubeletConfiguration) != 0 { - for customArg := range customKubeletConfiguration { - if _, ok := defaultKubeletConfiguration[customArg]; ok { - kubeletConfiguration[customArg] = customKubeletConfiguration[customArg] - delete(defaultKubeletConfiguration, customArg) - delete(customKubeletConfiguration, customArg) - } else { - kubeletConfiguration[customArg] = customKubeletConfiguration[customArg] - } - } - } - - if len(defaultKubeletConfiguration) != 0 { - for k, v := range defaultKubeletConfiguration { - kubeletConfiguration[k] = v - } - } - - if featureGates, ok := kubeletConfiguration["featureGates"].(map[string]bool); ok { - if versionutil.MustParseSemantic(kubeConf.Cluster.Kubernetes.Version).LessThan(versionutil.MustParseSemantic("v1.21.0")) { - delete(featureGates, "CSIStorageCapacity") - } - - if versionutil.MustParseSemantic(kubeConf.Cluster.Kubernetes.Version).AtLeast(versionutil.MustParseSemantic("v1.24.0")) { - delete(featureGates, "TTLAfterFinished") - } - - for k, v := range kubeConf.Cluster.Kubernetes.FeatureGates { - if _, ok := featureGates[k]; !ok { - featureGates[k] = v - } - } - } - - if kubeConf.Arg.Debug { - logger.Log.Debugf("Set kubeletConfiguration: %v", kubeletConfiguration) - } - - return kubeletConfiguration -} - -func GetKubeletCgroupDriver(runtime connector.Runtime, kubeConf *common.KubeConf) (string, error) { - var cmd, kubeletCgroupDriver string - switch kubeConf.Cluster.Kubernetes.ContainerManager { - case common.Docker, "": - cmd = "docker info | grep 'Cgroup Driver'" - case common.Crio: - cmd = "crio config | grep cgroup_manager" - case common.Containerd: - cmd = "containerd config dump | grep SystemdCgroup || echo 'SystemdCgroup = false'" - case common.Isula: - cmd = "isula info | grep 'Cgroup Driver'" - default: - kubeletCgroupDriver = "" - } - - checkResult, err := runtime.GetRunner().SudoCmd(cmd, false) - if err != nil { - return "", errors.Wrap(errors.WithStack(err), "Failed to get container runtime cgroup driver.") - } - if strings.Contains(checkResult, "systemd") || strings.Contains(checkResult, "SystemdCgroup = true") { - kubeletCgroupDriver = "systemd" - } else if strings.Contains(checkResult, "cgroupfs") || strings.Contains(checkResult, "SystemdCgroup = false") { - kubeletCgroupDriver = "cgroupfs" - } else { - return "", errors.Errorf("Failed to get container runtime cgroup driver from %s by run %s", checkResult, cmd) - } - return kubeletCgroupDriver, nil -} - -func GetKubeProxyConfiguration(kubeConf *common.KubeConf) map[string]interface{} { - defaultKubeProxyConfiguration := map[string]interface{}{ - "clusterCIDR": kubeConf.Cluster.Network.KubePodsCIDR, - "mode": kubeConf.Cluster.Kubernetes.ProxyMode, - "iptables": map[string]interface{}{ - "masqueradeAll": kubeConf.Cluster.Kubernetes.MasqueradeAll, - "masqueradeBit": 14, - "minSyncPeriod": "0s", - "syncPeriod": "30s", - }, - } - - customKubeProxyConfiguration := make(map[string]interface{}) - if len(kubeConf.Cluster.Kubernetes.KubeProxyConfiguration.Raw) != 0 { - err := yaml.Unmarshal(kubeConf.Cluster.Kubernetes.KubeProxyConfiguration.Raw, &customKubeProxyConfiguration) - if err != nil { - logger.Log.Fatal("failed to parse kube-proxy's configuration") - } - } - - kubeProxyConfiguration := make(map[string]interface{}) - if len(customKubeProxyConfiguration) != 0 { - for customArg := range customKubeProxyConfiguration { - if _, ok := defaultKubeProxyConfiguration[customArg]; ok { - kubeProxyConfiguration[customArg] = customKubeProxyConfiguration[customArg] - delete(defaultKubeProxyConfiguration, customArg) - delete(customKubeProxyConfiguration, customArg) - } else { - kubeProxyConfiguration[customArg] = customKubeProxyConfiguration[customArg] - } - } - } - - if len(defaultKubeProxyConfiguration) != 0 { - for defaultArg := range defaultKubeProxyConfiguration { - kubeProxyConfiguration[defaultArg] = defaultKubeProxyConfiguration[defaultArg] - } - } - - return kubeProxyConfiguration -} - -func copyStringMap(m map[string]string) map[string]string { - cp := make(map[string]string) - for k, v := range m { - cp[k] = v - } - - return cp -} diff --git a/cmd/kk/pkg/kubernetes/templates/kubelet_env.go b/cmd/kk/pkg/kubernetes/templates/kubelet_env.go deleted file mode 100644 index e5170eff5..000000000 --- a/cmd/kk/pkg/kubernetes/templates/kubelet_env.go +++ /dev/null @@ -1,38 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -// KubeletEnv defines the template of kubelet's Env for the kubelet's systemd service. -var KubeletEnv = template.Must(template.New("10-kubeadm.conf").Parse( - dedent.Dedent(`# Note: This dropin only works with kubeadm and kubelet v1.11+ -[Service] -Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" -Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" -# This is a file that "kubeadm init" and "kubeadm join" generate at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically -EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env -# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use -# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file. -EnvironmentFile=-/etc/default/kubelet -Environment="KUBELET_EXTRA_ARGS=--node-ip={{ .NodeIP }} --hostname-override={{ .Hostname }} {{ if .ContainerRuntime }}--network-plugin=cni{{ end }} {{range .KubeletArgs }} {{.}}{{ end }}" -ExecStart= -ExecStart=/usr/local/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS - `))) diff --git a/cmd/kk/pkg/kubernetes/templates/kubelet_service.go b/cmd/kk/pkg/kubernetes/templates/kubelet_service.go deleted file mode 100644 index d4752f672..000000000 --- a/cmd/kk/pkg/kubernetes/templates/kubelet_service.go +++ /dev/null @@ -1,40 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -// KubeletService defines the template of kubelete service for systemd. -var KubeletService = template.Must(template.New("kubelet.service").Parse( - dedent.Dedent(`[Unit] -Description=kubelet: The Kubernetes Node Agent -Documentation=http://kubernetes.io/docs/ - -[Service] -CPUAccounting=true -MemoryAccounting=true -ExecStart=/usr/local/bin/kubelet -Restart=always -StartLimitInterval=0 -RestartSec=10 - -[Install] -WantedBy=multi-user.target - `))) diff --git a/cmd/kk/pkg/kubernetes/upgrade_step_enum.go b/cmd/kk/pkg/kubernetes/upgrade_step_enum.go deleted file mode 100644 index 037b80ba3..000000000 --- a/cmd/kk/pkg/kubernetes/upgrade_step_enum.go +++ /dev/null @@ -1,40 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubernetes - -type UpgradeStep int - -const ( - ToV121 UpgradeStep = iota + 1 - ToV122 -) - -var UpgradeStepList = []UpgradeStep{ - ToV121, - ToV122, -} - -func (u UpgradeStep) String() string { - switch u { - case ToV121: - return "to v1.21" - case ToV122: - return "to v1.22" - default: - return "invalid option" - } -} diff --git a/cmd/kk/pkg/kubesphere/kubesphere_test.go b/cmd/kk/pkg/kubesphere/kubesphere_test.go deleted file mode 100644 index 46a63918d..000000000 --- a/cmd/kk/pkg/kubesphere/kubesphere_test.go +++ /dev/null @@ -1,99 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubesphere - -import ( - "reflect" - "testing" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere" -) - -func Test_mirrorRepo(t *testing.T) { - tests := []struct { - name string - version string - want string - }{ - { - name: "test_latest", - version: "latest", - want: "kubespheredev", - }, - { - name: "test_master", - version: "master", - want: "kubespheredev", - }, - { - name: "test_v3.2.1-rc.1", - version: "v3.2.1-rc.1", - want: "kubespheredev", - }, - { - name: "test_v3.2.1", - version: "v3.2.1", - want: "kubesphere", - }, - { - name: "test_v3.2.0", - version: "v3.2.0", - want: "kubesphere", - }, - { - name: "test_3.2.0", - version: "3.2.0", - want: "kubespheredev", - }, - { - name: "test_v3.2.0-alpha.1", - version: "v3.2.0-alpha.1", - want: "kubespheredev", - }, - { - name: "test_v1.2.0", - version: "v1.2.0", - want: "kubesphere", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := repo(tt.version) - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("StabledVersionSupport() got = %v, want %v", got, tt.want) - } - }) - } -} - -func repo(version string) string { - var r string - _, latest := kubesphere.LatestRelease(version) - _, dev := kubesphere.DevRelease(version) - _, stable := kubesphere.StabledVersionSupport(version) - switch { - case stable: - r = "kubesphere" - case dev: - r = "kubespheredev" - case latest: - r = "kubespheredev" - default: - r = "kubesphere" - } - return r -} diff --git a/cmd/kk/pkg/kubesphere/modules.go b/cmd/kk/pkg/kubesphere/modules.go deleted file mode 100644 index f7628f3c6..000000000 --- a/cmd/kk/pkg/kubesphere/modules.go +++ /dev/null @@ -1,246 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubesphere - -import ( - "fmt" - "os" - "path/filepath" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere/templates" -) - -type DeployModule struct { - common.KubeModule - Skip bool -} - -func (d *DeployModule) IsSkip() bool { - return d.Skip -} - -func (d *DeployModule) Init() { - d.Name = "DeployKubeSphereModule" - d.Desc = "Deploy KubeSphere" - - generateManifests := &task.RemoteTask{ - Name: "GenerateKsInstallerCRD", - Desc: "Generate KubeSphere ks-installer crd manifests", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(NotEqualDesiredVersion), - }, - Action: &action.Template{ - Template: templates.KsInstaller, - Dst: filepath.Join(common.KubeAddonsDir, templates.KsInstaller.Name()), - Data: util.Data{ - "Repo": MirrorRepo(d.KubeConf), - "Tag": d.KubeConf.Cluster.KubeSphere.Version, - }, - }, - Parallel: true, - } - - addConfig := &task.RemoteTask{ - Name: "AddKsInstallerConfig", - Desc: "Add config to ks-installer manifests", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(NotEqualDesiredVersion), - }, - Action: new(AddInstallerConfig), - Parallel: true, - } - - createNamespace := &task.RemoteTask{ - Name: "CreateKubeSphereNamespace", - Desc: "Create the kubesphere namespace", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(NotEqualDesiredVersion), - }, - Action: new(CreateNamespace), - Parallel: true, - } - - setup := &task.RemoteTask{ - Name: "SetupKsInstallerConfig", - Desc: "Setup ks-installer config", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(NotEqualDesiredVersion), - }, - Action: new(Setup), - Parallel: true, - } - - apply := &task.RemoteTask{ - Name: "ApplyKsInstaller", - Desc: "Apply ks-installer", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(NotEqualDesiredVersion), - }, - Action: new(Apply), - Parallel: true, - } - - d.Tasks = []task.Interface{ - generateManifests, - // apply crd installer.kubesphere.io/v1alpha1 - apply, - addConfig, - createNamespace, - setup, - apply, - } -} - -func MirrorRepo(kubeConf *common.KubeConf) string { - repo := kubeConf.Cluster.Registry.PrivateRegistry - namespaceOverride := kubeConf.Cluster.Registry.NamespaceOverride - version := kubeConf.Cluster.KubeSphere.Version - - _, ok := kubesphere.CNSource[version] - if ok && os.Getenv("KKZONE") == "cn" { - if repo == "" { - repo = "registry.cn-beijing.aliyuncs.com/kubesphereio" - } else if len(namespaceOverride) != 0 { - repo = fmt.Sprintf("%s/%s", repo, namespaceOverride) - } else { - repo = fmt.Sprintf("%s/kubesphere", repo) - } - } else { - if repo == "" { - _, latest := kubesphere.LatestRelease(version) - _, dev := kubesphere.DevRelease(version) - _, stable := kubesphere.StabledVersionSupport(version) - switch { - case stable: - repo = "kubesphere" - case dev: - repo = "kubespheredev" - case latest: - repo = "kubespheredev" - default: - repo = "kubesphere" - } - } else if len(namespaceOverride) != 0 { - repo = fmt.Sprintf("%s/%s", repo, namespaceOverride) - } else { - repo = fmt.Sprintf("%s/kubesphere", repo) - } - } - return repo -} - -type CheckResultModule struct { - common.KubeModule - Skip bool -} - -func (c *CheckResultModule) IsSkip() bool { - return c.Skip -} - -func (c *CheckResultModule) Init() { - c.Name = "CheckResultModule" - c.Desc = "Check deploy KubeSphere result" - - check := &task.RemoteTask{ - Name: "CheckKsInstallerResult", - Desc: "Check ks-installer result", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(NotEqualDesiredVersion), - }, - Action: new(Check), - Parallel: true, - } - - c.Tasks = []task.Interface{ - check, - } -} - -type CleanClusterConfigurationModule struct { - common.KubeModule - Skip bool -} - -func (c *CleanClusterConfigurationModule) IsSkip() bool { - return c.Skip -} - -func (c *CleanClusterConfigurationModule) Init() { - c.Name = "CleanClusterConfigurationModule" - c.Desc = "Clean redundant ClusterConfiguration config" - - // ensure there is no cc config, and prevent to reset cc config when upgrade the cluster - clean := &task.LocalTask{ - Name: "CleanClusterConfiguration", - Desc: "Clean redundant ClusterConfiguration config", - Action: new(CleanCC), - } - - c.Tasks = []task.Interface{ - clean, - } -} - -type ConvertModule struct { - common.KubeModule - Skip bool -} - -func (c *ConvertModule) IsSkip() bool { - return c.Skip -} - -func (c *ConvertModule) Init() { - c.Name = "ConvertModule" - c.Desc = "Convert ks-installer config v2 to v3" - - convert := &task.RemoteTask{ - Name: "ConvertV2ToV3", - Desc: "Convert ks-installer config v2 to v3", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(NotEqualDesiredVersion), - new(VersionBelowV3), - }, - Action: new(ConvertV2ToV3), - Parallel: true, - } - - c.Tasks = []task.Interface{ - convert, - } -} diff --git a/cmd/kk/pkg/kubesphere/prepares.go b/cmd/kk/pkg/kubesphere/prepares.go deleted file mode 100644 index 44dc28b17..000000000 --- a/cmd/kk/pkg/kubesphere/prepares.go +++ /dev/null @@ -1,58 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubesphere - -import ( - "github.com/pkg/errors" - versionutil "k8s.io/apimachinery/pkg/util/version" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type VersionBelowV3 struct { - common.KubePrepare -} - -func (v *VersionBelowV3) PreCheck(runtime connector.Runtime) (bool, error) { - versionStr, ok := v.PipelineCache.GetMustString(common.KubeSphereVersion) - if !ok { - return false, errors.New("get current kubesphere version failed by pipeline cache") - } - version := versionutil.MustParseSemantic(versionStr) - v300 := versionutil.MustParseSemantic("v3.0.0") - if v.KubeConf.Cluster.KubeSphere.Enabled && v.KubeConf.Cluster.KubeSphere.Version == "v3.0.0" && version.LessThan(v300) { - return true, nil - } - return false, nil -} - -type NotEqualDesiredVersion struct { - common.KubePrepare -} - -func (n *NotEqualDesiredVersion) PreCheck(runtime connector.Runtime) (bool, error) { - ksVersion, ok := n.PipelineCache.GetMustString(common.KubeSphereVersion) - if !ok { - ksVersion = "" - } - - if n.KubeConf.Cluster.KubeSphere.Version == ksVersion { - return false, nil - } - return true, nil -} diff --git a/cmd/kk/pkg/kubesphere/tasks.go b/cmd/kk/pkg/kubesphere/tasks.go deleted file mode 100644 index e867fb4ec..000000000 --- a/cmd/kk/pkg/kubesphere/tasks.go +++ /dev/null @@ -1,433 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubesphere - -import ( - "context" - "encoding/base64" - "fmt" - "os" - "path/filepath" - "strings" - "time" - - "github.com/pkg/errors" - yamlV3 "gopkg.in/yaml.v3" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - ksv2 "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubesphere/v2" - ksv3 "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubesphere/v3" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere/templates" -) - -type AddInstallerConfig struct { - common.KubeAction -} - -func (a *AddInstallerConfig) Execute(runtime connector.Runtime) error { - configurationBase64 := base64.StdEncoding.EncodeToString([]byte(a.KubeConf.Cluster.KubeSphere.Configurations)) - if _, err := runtime.GetRunner().SudoCmd( - fmt.Sprintf("echo %s | base64 -d >> /etc/kubernetes/addons/kubesphere.yaml", configurationBase64), - false); err != nil { - return errors.Wrap(errors.WithStack(err), "add config to ks-installer manifests failed") - } - return nil -} - -type CreateNamespace struct { - common.KubeAction -} - -func (c *CreateNamespace) Execute(runtime connector.Runtime) error { - _, err := runtime.GetRunner().SudoCmd(`cat <>--->", - ) - - fmt.Printf("%s \033[K\n", output) - time.Sleep(time.Duration(200) * time.Millisecond) - } else { - fmt.Printf("\033[%dA\033[K", position) - - output := fmt.Sprintf( - "%s%s%s", - notes, - strings.Repeat(" ", 10-i), - "<---<<", - ) - - fmt.Printf("%s \033[K\n", output) - time.Sleep(time.Duration(200) * time.Millisecond) - } - } - } - } - return nil -} - -func CheckKubeSphereStatus(ctx context.Context, runtime connector.Runtime, stopChan chan string) { - defer close(stopChan) - for { - select { - case <-ctx.Done(): - stopChan <- "" - default: - _, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl exec -n kubesphere-system "+ - "$(kubectl get pod -n kubesphere-system -l app=ks-installer -o jsonpath='{.items[0].metadata.name}') "+ - "-- ls /kubesphere/playbooks/kubesphere_running", false) - if err == nil { - output, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl exec -n kubesphere-system "+ - "$(kubectl get pod -n kubesphere-system -l app=ks-installer -o jsonpath='{.items[0].metadata.name}') "+ - "-- cat /kubesphere/playbooks/kubesphere_running", false) - if err == nil && output != "" { - stopChan <- output - break - } - } - } - } -} - -type CleanCC struct { - common.KubeAction -} - -func (c *CleanCC) Execute(runtime connector.Runtime) error { - c.KubeConf.Cluster.KubeSphere.Configurations = "\n" - return nil -} - -type ConvertV2ToV3 struct { - common.KubeAction -} - -func (c *ConvertV2ToV3) Execute(runtime connector.Runtime) error { - configV2Str, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl get cm -n kubesphere-system ks-installer -o jsonpath='{.data.ks-config\\.yaml}'", - false) - if err != nil { - return err - } - - clusterCfgV2 := ksv2.V2{} - clusterCfgV3 := ksv3.V3{} - if err := yamlV3.Unmarshal([]byte(configV2Str), &clusterCfgV2); err != nil { - return err - } - - configV3, err := MigrateConfig2to3(&clusterCfgV2, &clusterCfgV3) - if err != nil { - return err - } - c.KubeConf.Cluster.KubeSphere.Configurations = "---\n" + configV3 - return nil -} - -func MigrateConfig2to3(v2 *ksv2.V2, v3 *ksv3.V3) (string, error) { - v3.Etcd = ksv3.Etcd(v2.Etcd) - v3.Persistence = ksv3.Persistence(v2.Persistence) - v3.Alerting = ksv3.Alerting(v2.Alerting) - v3.Notification = ksv3.Notification(v2.Notification) - v3.LocalRegistry = v2.LocalRegistry - v3.Servicemesh = ksv3.Servicemesh(v2.Servicemesh) - v3.Devops = ksv3.Devops(v2.Devops) - v3.Openpitrix = ksv3.Openpitrix(v2.Openpitrix) - v3.Console = ksv3.Console(v2.Console) - - if v2.MetricsServerNew.Enabled == "" { - if v2.MetricsServerOld.Enabled == "true" || v2.MetricsServerOld.Enabled == "True" { - v3.MetricsServer.Enabled = true - } else { - v3.MetricsServer.Enabled = false - } - } else { - if v2.MetricsServerNew.Enabled == "true" || v2.MetricsServerNew.Enabled == "True" { - v3.MetricsServer.Enabled = true - } else { - v3.MetricsServer.Enabled = false - } - } - - v3.Monitoring.PrometheusMemoryRequest = v2.Monitoring.PrometheusMemoryRequest - //v3.Monitoring.PrometheusReplicas = v2.Monitoring.PrometheusReplicas - v3.Monitoring.PrometheusVolumeSize = v2.Monitoring.PrometheusVolumeSize - //v3.Monitoring.AlertmanagerReplicas = 1 - - v3.Common.EtcdVolumeSize = v2.Common.EtcdVolumeSize - v3.Common.MinioVolumeSize = v2.Common.MinioVolumeSize - v3.Common.MysqlVolumeSize = v2.Common.MysqlVolumeSize - v3.Common.OpenldapVolumeSize = v2.Common.OpenldapVolumeSize - v3.Common.RedisVolumSize = v2.Common.RedisVolumSize - //v3.Common.ES.ElasticsearchDataReplicas = v2.Logging.ElasticsearchDataReplicas - //v3.Common.ES.ElasticsearchMasterReplicas = v2.Logging.ElasticsearchMasterReplicas - v3.Common.ES.ElkPrefix = v2.Logging.ElkPrefix - v3.Common.ES.LogMaxAge = v2.Logging.LogMaxAge - if v2.Logging.ElasticsearchVolumeSize == "" { - v3.Common.ES.ElasticsearchDataVolumeSize = v2.Logging.ElasticsearchDataVolumeSize - v3.Common.ES.ElasticsearchMasterVolumeSize = v2.Logging.ElasticsearchMasterVolumeSize - } else { - v3.Common.ES.ElasticsearchMasterVolumeSize = "4Gi" - v3.Common.ES.ElasticsearchDataVolumeSize = v2.Logging.ElasticsearchVolumeSize - } - - v3.Logging.Enabled = v2.Logging.Enabled - v3.Logging.LogsidecarReplicas = v2.Logging.LogsidecarReplicas - - v3.Authentication.JwtSecret = "" - v3.Multicluster.ClusterRole = "none" - v3.Events.Ruler.Replicas = 2 - - var clusterConfiguration = ksv3.ClusterConfig{ - ApiVersion: "installer.kubesphere.io/v1alpha1", - Kind: "ClusterConfiguration", - Metadata: ksv3.Metadata{ - Name: "ks-installer", - Namespace: "kubesphere-system", - Label: ksv3.Label{Version: "v3.0.0"}, - }, - Spec: v3, - } - - configV3, err := yamlV3.Marshal(clusterConfiguration) - if err != nil { - return "", err - } - - return string(configV3), nil -} diff --git a/cmd/kk/pkg/kubesphere/v2/types.go b/cmd/kk/pkg/kubesphere/v2/types.go deleted file mode 100644 index 9d3ce32d7..000000000 --- a/cmd/kk/pkg/kubesphere/v2/types.go +++ /dev/null @@ -1,109 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v2 - -type V2 struct { - Persistence Persistence `yaml:"persistence"` - Common Common `yaml:"common"` - Etcd Etcd `yaml:"etcd"` - MetricsServerOld MetricsServerOld `yaml:"metrics-server"` - MetricsServerNew MetricsServerNew `yaml:"metrics_server"` - Console Console `yaml:"console"` - Monitoring Monitoring `yaml:"monitoring"` - Logging Logging `yaml:"logging"` - Openpitrix Openpitrix `yaml:"openpitrix"` - Devops Devops `yaml:"devops"` - Servicemesh Servicemesh `yaml:"servicemesh"` - Notification Notification `yaml:"notification"` - Alerting Alerting `yaml:"alerting"` - LocalRegistry string `yaml:"local_registry"` -} - -type Persistence struct { - StorageClass string `yaml:"storageClass"` -} -type Etcd struct { - Monitoring bool `yaml:"monitoring"` - EndpointIps string `yaml:"endpointIps"` - Port int `yaml:"port"` - TlsEnable bool `yaml:"tlsEnable"` -} - -type Common struct { - MysqlVolumeSize string `yaml:"mysqlVolumeSize"` - MinioVolumeSize string `yaml:"minioVolumeSize"` - EtcdVolumeSize string `yaml:"etcdVolumeSize"` - OpenldapVolumeSize string `yaml:"openldapVolumeSize"` - RedisVolumSize string `yaml:"redisVolumSize"` -} - -type MetricsServerOld struct { - Enabled string `yaml:"enabled"` -} - -type MetricsServerNew struct { - Enabled string `yaml:"enabled"` -} - -type Console struct { - EnableMultiLogin bool `yaml:"enableMultiLogin"` - Port int `yaml:"port"` -} - -type Monitoring struct { - PrometheusReplicas int `yaml:"prometheusReplicas"` - PrometheusMemoryRequest string `yaml:"prometheusMemoryRequest"` - PrometheusVolumeSize string `yaml:"prometheusVolumeSize"` -} - -type Logging struct { - Enabled bool `yaml:"enabled"` - ElasticsearchMasterReplicas int `yaml:"elasticsearchMasterReplicas"` - ElasticsearchDataReplicas int `yaml:"elasticsearchDataReplicas"` - LogsidecarReplicas int `yaml:"logsidecarReplicas"` - ElasticsearchVolumeSize string `yaml:"elasticsearchVolumeSize"` - ElasticsearchMasterVolumeSize string `yaml:"elasticsearchMasterVolumeSize"` - ElasticsearchDataVolumeSize string `yaml:"elasticsearchDataVolumeSize"` - LogMaxAge int `yaml:"logMaxAge"` - ElkPrefix string `yaml:"elkPrefix"` -} - -type Openpitrix struct { - Enabled bool `yaml:"enabled"` -} - -type Devops struct { - Enabled bool `yaml:"enabled"` - JenkinsMemoryLim string `yaml:"jenkinsMemoryLim"` - JenkinsMemoryReq string `yaml:"jenkinsMemoryReq"` - JenkinsVolumeSize string `yaml:"jenkinsVolumeSize"` - JenkinsjavaoptsXms string `yaml:"jenkinsJavaOpts_Xms"` - JenkinsjavaoptsXmx string `yaml:"jenkinsJavaOpts_Xmx"` - JenkinsjavaoptsMaxram string `yaml:"jenkinsJavaOpts_MaxRAM"` -} - -type Servicemesh struct { - Enabled bool `yaml:"enabled"` -} - -type Notification struct { - Enabled bool `yaml:"enabled"` -} - -type Alerting struct { - Enabled bool `yaml:"enabled"` -} diff --git a/cmd/kk/pkg/kubesphere/v3/types.go b/cmd/kk/pkg/kubesphere/v3/types.go deleted file mode 100644 index 97b97198f..000000000 --- a/cmd/kk/pkg/kubesphere/v3/types.go +++ /dev/null @@ -1,160 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v3 - -type ClusterConfig struct { - ApiVersion string `yaml:"apiVersion"` - Kind string `yaml:"kind"` - Metadata Metadata `yaml:"metadata"` - Spec *V3 `yaml:"spec"` -} - -type Metadata struct { - Name string `yaml:"name"` - Namespace string `yaml:"namespace"` - Label Label `yaml:"labels"` -} - -type Label struct { - Version string `yaml:"version"` -} - -type V3 struct { - Persistence Persistence `yaml:"persistence"` - Authentication Authentication `yaml:"authentication"` - Common Common `yaml:"common"` - Etcd Etcd `yaml:"etcd"` - MetricsServer MetricsServer `yaml:"metrics_server"` - Console Console `yaml:"console"` - Monitoring Monitoring `yaml:"monitoring"` - Logging Logging `yaml:"logging"` - Openpitrix Openpitrix `yaml:"openpitrix"` - Devops Devops `yaml:"devops"` - Servicemesh Servicemesh `yaml:"servicemesh"` - Notification Notification `yaml:"notification"` - Alerting Alerting `yaml:"alerting"` - Auditing Auditing `yaml:"auditing"` - Events Events `yaml:"events"` - Multicluster Multicluster `yaml:"multicluster"` - Networkpolicy Networkpolicy `yaml:"networkpolicy"` - LocalRegistry string `yaml:"local_registry"` -} - -type Persistence struct { - StorageClass string `yaml:"storageClass"` -} - -type MetricsServer struct { - Enabled bool `yaml:"enabled"` -} - -type Authentication struct { - JwtSecret string `yaml:"jwtSecret"` -} -type Etcd struct { - Monitoring bool `yaml:"monitoring"` - EndpointIps string `yaml:"endpointIps"` - Port int `yaml:"port"` - TlsEnable bool `yaml:"tlsEnable"` -} - -type Common struct { - MysqlVolumeSize string `yaml:"mysqlVolumeSize"` - MinioVolumeSize string `yaml:"minioVolumeSize"` - EtcdVolumeSize string `yaml:"etcdVolumeSize"` - OpenldapVolumeSize string `yaml:"openldapVolumeSize"` - RedisVolumSize string `yaml:"redisVolumSize"` - ES ES `yaml:"es"` -} - -type ES struct { - //ElasticsearchMasterReplicas int `yaml:"elasticsearchMasterReplicas"` - //ElasticsearchDataReplicas int `yaml:"elasticsearchDataReplicas"` - ElasticsearchMasterVolumeSize string `yaml:"elasticsearchMasterVolumeSize"` - ElasticsearchDataVolumeSize string `yaml:"elasticsearchDataVolumeSize"` - LogMaxAge int `yaml:"logMaxAge"` - ElkPrefix string `yaml:"elkPrefix"` -} - -type Console struct { - EnableMultiLogin bool `yaml:"enableMultiLogin"` - Port int `yaml:"port"` -} - -type Alerting struct { - Enabled bool `yaml:"enabled"` -} - -type Auditing struct { - Enabled bool `yaml:"enabled"` -} - -type Devops struct { - Enabled bool `yaml:"enabled"` - JenkinsMemoryLim string `yaml:"jenkinsMemoryLim"` - JenkinsMemoryReq string `yaml:"jenkinsMemoryReq"` - JenkinsVolumeSize string `yaml:"jenkinsVolumeSize"` - JenkinsjavaoptsXms string `yaml:"jenkinsJavaOpts_Xms"` - JenkinsjavaoptsXmx string `yaml:"jenkinsJavaOpts_Xmx"` - JenkinsjavaoptsMaxram string `yaml:"jenkinsJavaOpts_MaxRAM"` -} - -type Events struct { - Enabled bool `yaml:"enabled"` - Ruler Ruler `yaml:"ruler"` -} - -type Ruler struct { - Enabled bool `yaml:"enabled"` - Replicas int `yaml:"replicas"` -} - -type Logging struct { - Enabled bool `yaml:"enabled"` - LogsidecarReplicas int `yaml:"logsidecarReplicas"` -} - -type Metrics struct { - Enabled bool `yaml:"enabled"` -} - -type Monitoring struct { - //AlertmanagerReplicas int `yaml:"alertmanagerReplicas"` - //PrometheusReplicas int `yaml:"prometheusReplicas"` - PrometheusMemoryRequest string `yaml:"prometheusMemoryRequest"` - PrometheusVolumeSize string `yaml:"prometheusVolumeSize"` -} - -type Multicluster struct { - ClusterRole string `yaml:"clusterRole"` -} - -type Networkpolicy struct { - Enabled bool `yaml:"enabled"` -} - -type Notification struct { - Enabled bool `yaml:"enabled"` -} - -type Openpitrix struct { - Enabled bool `yaml:"enabled"` -} - -type Servicemesh struct { - Enabled bool `yaml:"enabled"` -} diff --git a/cmd/kk/pkg/loadbalancer/common.go b/cmd/kk/pkg/loadbalancer/common.go deleted file mode 100644 index d53b16fca..000000000 --- a/cmd/kk/pkg/loadbalancer/common.go +++ /dev/null @@ -1,21 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package loadbalancer - -const ( - LocalServer = "server: https://127.0.0.1" -) diff --git a/cmd/kk/pkg/loadbalancer/module.go b/cmd/kk/pkg/loadbalancer/module.go deleted file mode 100644 index 7492581f3..000000000 --- a/cmd/kk/pkg/loadbalancer/module.go +++ /dev/null @@ -1,380 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package loadbalancer - -import ( - "path/filepath" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/loadbalancer/templates" -) - -type HaproxyModule struct { - common.KubeModule - Skip bool -} - -func (h *HaproxyModule) IsSkip() bool { - return h.Skip -} - -func (h *HaproxyModule) Init() { - h.Name = "InternalLoadbalancerModule" - h.Desc = "Install internal load balancer" - - haproxyCfg := &task.RemoteTask{ - Name: "GenerateHaproxyConfig", - Desc: "Generate haproxy.cfg", - Hosts: h.Runtime.GetHostsByRole(common.Worker), - Prepare: new(common.OnlyWorker), - Action: &action.Template{ - Template: templates.HaproxyConfig, - Dst: filepath.Join(common.HaproxyDir, templates.HaproxyConfig.Name()), - Data: util.Data{ - "MasterNodes": templates.MasterNodeStr(h.Runtime, h.KubeConf), - "LoadbalancerApiserverPort": kubekeyapiv1alpha2.DefaultApiserverPort, - "LoadbalancerApiserverHealthcheckPort": 8081, - "KubernetesType": h.KubeConf.Cluster.Kubernetes.Type, - }, - }, - Parallel: true, - } - - // Calculation config md5 as the checksum. - // It will make load balancer reload when config changes. - getMd5Sum := &task.RemoteTask{ - Name: "GetChecksumFromConfig", - Desc: "Calculate the MD5 value according to haproxy.cfg", - Hosts: h.Runtime.GetHostsByRole(common.Worker), - Prepare: new(common.OnlyWorker), - Action: new(GetChecksum), - Parallel: true, - } - - haproxyManifestK8s := &task.RemoteTask{ - Name: "GenerateHaproxyManifest", - Desc: "Generate haproxy manifest", - Hosts: h.Runtime.GetHostsByRole(common.Worker), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyWorker), - new(common.OnlyKubernetes), - }, - Action: new(GenerateHaproxyManifest), - Parallel: true, - } - - // UpdateKubeletConfig Update server field in kubelet.conf - // When create a HA cluster by internal LB, we will set the server filed to 127.0.0.1:6443 (default) which in kubelet.conf. - // Because of that, the control plone node's kubelet connect the local api-server. - // And the work node's kubelet connect 127.0.0.1:6443 (default) that is proxy by the node's local nginx. - updateKubeletConfig := &task.RemoteTask{ - Name: "UpdateKubeletConfig", - Desc: "Update kubelet config", - Hosts: h.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyKubernetes), - new(updateKubeletPrepare), - }, - Action: new(UpdateKubelet), - Parallel: true, - Retry: 3, - } - - // updateKubeProxyConfig is used to update kube-proxy configmap and restart tge kube-proxy pod. - updateKubeProxyConfig := &task.RemoteTask{ - Name: "UpdateKubeProxyConfig", - Desc: "Update kube-proxy configmap", - Hosts: []connector.Host{h.Runtime.GetHostsByRole(common.Master)[0]}, - Prepare: &prepare.PrepareCollection{ - new(common.EnableKubeProxy), - new(common.OnlyKubernetes), - new(common.OnlyFirstMaster), - new(updateKubeProxyPrapre), - }, - Action: new(UpdateKubeProxy), - Parallel: true, - Retry: 3, - } - - // UpdateHostsFile is used to update the '/etc/hosts'. Make the 'lb.kubesphere.local' address to set as 127.0.0.1. - // All of the 'admin.conf' and '/.kube/config' will connect to 127.0.0.1:6443. - updateHostsFile := &task.RemoteTask{ - Name: "UpdateHostsFile", - Desc: "Update /etc/hosts", - Hosts: h.Runtime.GetHostsByRole(common.K8s), - Action: new(UpdateHosts), - Parallel: true, - Retry: 3, - } - - h.Tasks = []task.Interface{ - haproxyCfg, - getMd5Sum, - haproxyManifestK8s, - updateKubeletConfig, - updateKubeProxyConfig, - updateHostsFile, - } -} - -type KubevipModule struct { - common.KubeModule - Skip bool -} - -func (k *KubevipModule) IsSkip() bool { - return k.Skip -} - -func (k *KubevipModule) Init() { - k.Name = "InternalLoadbalancerModule" - k.Desc = "Install internal load balancer" - - checkVIPAddress := &task.RemoteTask{ - Name: "CheckVIPAddress", - Desc: "Check VIP Address", - Hosts: k.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(CheckVIPAddress), - Parallel: true, - } - - getInterface := &task.RemoteTask{ - Name: "GetNodeInterface", - Desc: "Get Node Interface", - Hosts: k.Runtime.GetHostsByRole(common.Master), - Action: new(GetInterfaceName), - Parallel: true, - } - - kubevipManifestOnlyFirstMaster := &task.RemoteTask{ - Name: "GenerateKubevipManifest", - Desc: "Generate kubevip manifest at first master", - Hosts: k.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(GenerateKubevipManifest), - Parallel: true, - } - - kubevipManifestNotFirstMaster := &task.RemoteTask{ - Name: "GenerateKubevipManifest", - Desc: "Generate kubevip manifest at other master", - Hosts: k.Runtime.GetHostsByRole(common.Master), - Prepare: &common.OnlyFirstMaster{Not: true}, - Action: new(GenerateKubevipManifest), - Parallel: true, - } - - if exist, _ := k.BaseModule.PipelineCache.GetMustBool(common.ClusterExist); exist { - k.Tasks = []task.Interface{ - checkVIPAddress, - getInterface, - kubevipManifestNotFirstMaster, - } - } else { - k.Tasks = []task.Interface{ - checkVIPAddress, - getInterface, - kubevipManifestOnlyFirstMaster, - } - } -} - -type K3sHaproxyModule struct { - common.KubeModule - Skip bool -} - -func (k *K3sHaproxyModule) IsSkip() bool { - return k.Skip -} - -func (k *K3sHaproxyModule) Init() { - k.Name = "InternalLoadbalancerModule" - k.Name = "Install internal load balancer" - - haproxyCfg := &task.RemoteTask{ - Name: "GenerateHaproxyConfig", - Desc: "Generate haproxy.cfg", - Hosts: k.Runtime.GetHostsByRole(common.Worker), - Prepare: new(common.OnlyWorker), - Action: &action.Template{ - Template: templates.HaproxyConfig, - Dst: filepath.Join(common.HaproxyDir, templates.HaproxyConfig.Name()), - Data: util.Data{ - "MasterNodes": templates.MasterNodeStr(k.Runtime, k.KubeConf), - "LoadbalancerApiserverPort": k.KubeConf.Cluster.ControlPlaneEndpoint.Port, - "LoadbalancerApiserverHealthcheckPort": 8081, - "KubernetesType": k.KubeConf.Cluster.Kubernetes.Type, - }, - }, - Parallel: true, - } - - // Calculation config md5 as the checksum. - // It will make load balancer reload when config changes. - getMd5Sum := &task.RemoteTask{ - Name: "GetChecksumFromConfig", - Desc: "Calculate the MD5 value according to haproxy.cfg", - Hosts: k.Runtime.GetHostsByRole(common.Worker), - Prepare: new(common.OnlyWorker), - Action: new(GetChecksum), - Parallel: true, - } - - haproxyManifestK3s := &task.RemoteTask{ - Name: "GenerateHaproxyManifestK3s", - Desc: "Generate haproxy manifest", - Hosts: k.Runtime.GetHostsByRole(common.Worker), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyWorker), - new(common.OnlyK3s), - }, - Action: new(GenerateK3sHaproxyManifest), - Parallel: true, - } - - updateK3sConfig := &task.RemoteTask{ - Name: "UpdateK3sConfig", - Desc: "Update k3s config", - Hosts: k.Runtime.GetHostsByRole(common.Worker), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyK3s), - new(updateK3sPrepare), - }, - Action: new(UpdateK3s), - Parallel: true, - Retry: 3, - } - - // UpdateHostsFile is used to update the '/etc/hosts'. Make the 'lb.kubesphere.local' address to set as 127.0.0.1. - // All of the 'admin.conf' and '/.kube/config' will connect to 127.0.0.1:6443. - updateHostsFile := &task.RemoteTask{ - Name: "UpdateHostsFile", - Desc: "Update /etc/hosts", - Hosts: k.Runtime.GetHostsByRole(common.K8s), - Action: new(UpdateHosts), - Parallel: true, - Retry: 3, - } - - k.Tasks = []task.Interface{ - haproxyCfg, - getMd5Sum, - haproxyManifestK3s, - updateK3sConfig, - updateHostsFile, - } -} - -type K3sKubevipModule struct { - common.KubeModule - Skip bool -} - -func (k *K3sKubevipModule) IsSkip() bool { - return k.Skip -} - -func (k *K3sKubevipModule) Init() { - k.Name = "InternalLoadbalancerModule" - k.Name = "Install internal load balancer" - - checkVIPAddress := &task.RemoteTask{ - Name: "CheckVIPAddress", - Desc: "Check VIP Address", - Hosts: k.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(CheckVIPAddress), - Parallel: true, - } - - createManifestsFolder := &task.RemoteTask{ - Name: "CreateManifestsFolder", - Desc: "Create Manifests Folder", - Hosts: k.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(CreateManifestsFolder), - Parallel: true, - } - - getInterface := &task.RemoteTask{ - Name: "GetNodeInterface", - Desc: "Get Node Interface", - Hosts: k.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(GetInterfaceName), - Parallel: true, - } - - kubevipDaemonsetK3s := &task.RemoteTask{ - Name: "GenerateKubevipManifest", - Desc: "Generate kubevip daemoset", - Hosts: k.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(GenerateK3sKubevipDaemonset), - Parallel: true, - } - - k.Tasks = []task.Interface{ - checkVIPAddress, - createManifestsFolder, - getInterface, - kubevipDaemonsetK3s, - } -} - -type DeleteVIPModule struct { - common.KubeModule - Skip bool -} - -func (k *DeleteVIPModule) IsSkip() bool { - return k.Skip -} - -func (k *DeleteVIPModule) Init() { - k.Name = "DeleteVIPModule" - k.Desc = "Delete VIP" - - getInterface := &task.RemoteTask{ - Name: "GetNodeInterface", - Desc: "Get Node Interface", - Hosts: k.Runtime.GetHostsByRole(common.Master), - Action: new(GetInterfaceName), - Parallel: true, - } - - DeleteVIP := &task.RemoteTask{ - Name: "Delete VIP", - Desc: "Delete the VIP", - Hosts: k.Runtime.GetHostsByRole(common.Master), - Action: new(DeleteVIP), - Parallel: true, - } - - k.Tasks = []task.Interface{ - getInterface, - DeleteVIP, - } -} diff --git a/cmd/kk/pkg/loadbalancer/prepares.go b/cmd/kk/pkg/loadbalancer/prepares.go deleted file mode 100644 index 7dacbf4dc..000000000 --- a/cmd/kk/pkg/loadbalancer/prepares.go +++ /dev/null @@ -1,95 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package loadbalancer - -import ( - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" -) - -type updateK3sPrepare struct { - common.KubePrepare -} - -func (u *updateK3sPrepare) PreCheck(runtime connector.Runtime) (bool, error) { - exist, err := runtime.GetRunner().FileExist("/etc/systemd/system/k3s.service") - if err != nil { - return false, err - } - - if exist { - if out, err := runtime.GetRunner().SudoCmd("sed -n '/--server=.*/p' /etc/systemd/system/k3s.service", false); err != nil { - return false, err - } else { - if strings.Contains(strings.TrimSpace(out), LocalServer) { - logger.Log.Debugf("do not restart kubelet, /etc/systemd/system/k3s.service content is %s", out) - return false, nil - } - } - } else { - return false, errors.New("Failed to find /etc/systemd/system/k3s.service") - } - return true, nil -} - -type updateKubeletPrepare struct { - common.KubePrepare -} - -func (u *updateKubeletPrepare) PreCheck(runtime connector.Runtime) (bool, error) { - exist, err := runtime.GetRunner().FileExist("/etc/kubernetes/kubelet.conf") - if err != nil { - return false, err - } - - if exist { - if out, err := runtime.GetRunner().SudoCmd("sed -n '/server:.*/p' /etc/kubernetes/kubelet.conf", true); err != nil { - return false, err - } else { - if strings.Contains(strings.TrimSpace(out), LocalServer) { - logger.Log.Debugf("do not restart kubelet, /etc/kubernetes/kubelet.conf content is %s", out) - return false, nil - } - } - } else { - return false, errors.New("Failed to find /etc/kubernetes/kubelet.conf") - } - return true, nil -} - -type updateKubeProxyPrapre struct { - common.KubePrepare -} - -func (u *updateKubeProxyPrapre) PreCheck(runtime connector.Runtime) (bool, error) { - if out, err := runtime.GetRunner().SudoCmd( - "set -o pipefail && /usr/local/bin/kubectl --kubeconfig /etc/kubernetes/admin.conf get configmap kube-proxy -n kube-system -o yaml "+ - "| sed -n '/server:.*/p'", false); err != nil { - return false, err - } else { - if strings.Contains(strings.TrimSpace(out), LocalServer) { - logger.Log.Debugf("do not restart kube-proxy, configmap kube-proxy content is %s", out) - return false, nil - } - } - return true, nil -} diff --git a/cmd/kk/pkg/loadbalancer/tasks.go b/cmd/kk/pkg/loadbalancer/tasks.go deleted file mode 100644 index c1087d57f..000000000 --- a/cmd/kk/pkg/loadbalancer/tasks.go +++ /dev/null @@ -1,324 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package loadbalancer - -import ( - "fmt" - "path/filepath" - "strconv" - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/loadbalancer/templates" -) - -type GetChecksum struct { - common.KubeAction -} - -func (g *GetChecksum) Execute(runtime connector.Runtime) error { - md5Str, err := runtime.GetRunner().FileMd5(filepath.Join(common.HaproxyDir, "haproxy.cfg")) - if err != nil { - return err - } - host := runtime.RemoteHost() - // type: string - host.GetCache().Set("md5", md5Str) - return nil -} - -type GenerateHaproxyManifest struct { - common.KubeAction -} - -func (g *GenerateHaproxyManifest) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - md5Str, ok := host.GetCache().GetMustString("md5") - if !ok { - return errors.New("get haproxy config md5 sum by host label failed") - } - - templateAction := action.Template{ - Template: templates.HaproxyManifest, - Dst: filepath.Join(common.KubeManifestDir, templates.HaproxyManifest.Name()), - Data: util.Data{ - "HaproxyImage": images.GetImage(runtime, g.KubeConf, "haproxy").ImageName(), - "HealthCheckPort": 8081, - "Checksum": md5Str, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type UpdateK3s struct { - common.KubeAction -} - -func (u *UpdateK3s) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("sed -i 's#--server=.*\"#--server=https://127.0.0.1:%s\"#g' /etc/systemd/system/k3s.service", false); err != nil { - return err - } - if _, err := runtime.GetRunner().SudoCmd("systemctl restart k3s", false); err != nil { - return err - } - return nil -} - -type UpdateKubelet struct { - common.KubeAction -} - -func (u *UpdateKubelet) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "sed -i 's#server:.*#server: https://127.0.0.1:%s#g' /etc/kubernetes/kubelet.conf", - strconv.Itoa(u.KubeConf.Cluster.ControlPlaneEndpoint.Port)), false); err != nil { - return err - } - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && systemctl restart kubelet", false); err != nil { - return err - } - return nil -} - -type UpdateKubeProxy struct { - common.KubeAction -} - -func (u *UpdateKubeProxy) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "set -o pipefail "+ - "&& /usr/local/bin/kubectl --kubeconfig /etc/kubernetes/admin.conf get configmap kube-proxy -n kube-system -o yaml "+ - "| sed 's#server:.*#server: https://127.0.0.1:%s#g' "+ - "| /usr/local/bin/kubectl --kubeconfig /etc/kubernetes/admin.conf replace -f -", - strconv.Itoa(u.KubeConf.Cluster.ControlPlaneEndpoint.Port)), false); err != nil { - return err - } - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl "+ - "--kubeconfig /etc/kubernetes/admin.conf delete pod "+ - "-n kube-system "+ - "-l k8s-app=kube-proxy "+ - "--force "+ - "--grace-period=0", false); err != nil { - return err - } - return nil -} - -type UpdateHosts struct { - common.KubeAction -} - -func (u *UpdateHosts) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("sed -i 's#.* %s#127.0.0.1 %s#g' /etc/hosts", - u.KubeConf.Cluster.ControlPlaneEndpoint.Domain, u.KubeConf.Cluster.ControlPlaneEndpoint.Domain), false); err != nil { - return err - } - return nil -} - -type CheckVIPAddress struct { - common.KubeAction -} - -func (c *CheckVIPAddress) Execute(runtime connector.Runtime) error { - if c.KubeConf.Cluster.ControlPlaneEndpoint.Address == "" { - return errors.New("VIP address is empty") - } else { - return nil - } -} - -type GetInterfaceName struct { - common.KubeAction -} - -func (g *GetInterfaceName) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - if g.KubeConf.Cluster.ControlPlaneEndpoint.KubeVip.Mode == "BGP" { - host.GetCache().Set("interface", "lo") - return nil - } - cmd := fmt.Sprintf("ip route "+ - "| grep ' %s ' "+ - "| grep 'proto kernel scope link src'"+ - "| sed -e \"s/^.*dev.//\" -e \"s/.proto.*//\""+ - "| uniq ", host.GetAddress()) - interfaceName, err := runtime.GetRunner().SudoCmd(cmd, false) - if err != nil { - return err - } - if interfaceName == "" { - return errors.New("get interface failed") - } - // type: string - host.GetCache().Set("interface", interfaceName) - return nil -} - -type GenerateKubevipManifest struct { - common.KubeAction -} - -func (g *GenerateKubevipManifest) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - interfaceName, ok := host.GetCache().GetMustString("interface") - if !ok { - return errors.New("get interface failed") - } - BGPMode := g.KubeConf.Cluster.ControlPlaneEndpoint.KubeVip.Mode == "BGP" - hosts := runtime.GetHostsByRole(common.Master) - var BGPPeersArr []string - for _, value := range hosts { - address := value.GetAddress() - if address == host.GetAddress() { - continue - } - BGPPeersArr = append(BGPPeersArr, fmt.Sprintf("%s:65000::false", address)) - } - BGPPeers := strings.Join(BGPPeersArr, ",") - templateAction := action.Template{ - Template: templates.KubevipManifest, - Dst: filepath.Join(common.KubeManifestDir, templates.KubevipManifest.Name()), - Data: util.Data{ - "BGPMode": BGPMode, - "VipInterface": interfaceName, - "BGPRouterID": host.GetAddress(), - "BGPPeers": BGPPeers, - "KubeVip": g.KubeConf.Cluster.ControlPlaneEndpoint.Address, - "KubevipImage": images.GetImage(runtime, g.KubeConf, "kubevip").ImageName(), - }, - } - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type GenerateK3sHaproxyManifest struct { - common.KubeAction -} - -func (g *GenerateK3sHaproxyManifest) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - md5Str, ok := host.GetCache().GetMustString("md5") - if !ok { - return errors.New("get haproxy config md5 sum by host label failed") - } - - templateAction := action.Template{ - Template: templates.HaproxyManifest, - Dst: filepath.Join("/var/lib/rancher/k3s/agent/pod-manifests", templates.HaproxyManifest.Name()), - Data: util.Data{ - "HaproxyImage": images.GetImage(runtime, g.KubeConf, "haproxy").ImageName(), - "HealthCheckPort": 8081, - "Checksum": md5Str, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type CreateManifestsFolder struct { - action.BaseAction -} - -func (h *CreateManifestsFolder) Execute(runtime connector.Runtime) error { - _, err := runtime.GetRunner().SudoCmd("mkdir -p /var/lib/rancher/k3s/server/manifests/", false) - if err != nil { - return err - } - return nil -} - -type GenerateK3sKubevipDaemonset struct { - common.KubeAction -} - -func (g *GenerateK3sKubevipDaemonset) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - interfaceName, ok := host.GetCache().GetMustString("interface") - if !ok { - return errors.New("get interface failed") - } - BGPMode := g.KubeConf.Cluster.ControlPlaneEndpoint.KubeVip.Mode == "BGP" - hosts := runtime.GetHostsByRole(common.Master) - var BGPPeersArr []string - for _, value := range hosts { - address := value.GetAddress() - if address == host.GetAddress() { - continue - } - BGPPeersArr = append(BGPPeersArr, fmt.Sprintf("%s:65000::false", address)) - } - BGPPeers := strings.Join(BGPPeersArr, ",") - templateAction := action.Template{ - Template: templates.K3sKubevipManifest, - Dst: filepath.Join("/var/lib/rancher/k3s/server/manifests/", templates.K3sKubevipManifest.Name()), - Data: util.Data{ - "BGPMode": BGPMode, - "KubeVipVersion": images.GetImage(runtime, g.KubeConf, "kubevip").Tag, - "VipInterface": interfaceName, - "BGPRouterID": host.GetAddress(), - "BGPPeers": BGPPeers, - "KubeVip": g.KubeConf.Cluster.ControlPlaneEndpoint.Address, - "KubevipImage": images.GetImage(runtime, g.KubeConf, "kubevip").ImageName(), - }, - } - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type DeleteVIP struct { - common.KubeAction -} - -func (g *DeleteVIP) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - interfaceName, ok := host.GetCache().GetMustString("interface") - if !ok { - return errors.New("get interface failed") - } - - address := host.GetAddress() - internalAddress := host.GetInternalAddress() - if address == g.KubeConf.Cluster.ControlPlaneEndpoint.Address || internalAddress == g.KubeConf.Cluster.ControlPlaneEndpoint.Address { - return nil - } - - cmd := fmt.Sprintf("ip addr del %s dev %s", g.KubeConf.Cluster.ControlPlaneEndpoint.Address, interfaceName) - runtime.GetRunner().SudoCmd(cmd, false) - return nil -} diff --git a/cmd/kk/pkg/loadbalancer/templates/haproxyConfig.go b/cmd/kk/pkg/loadbalancer/templates/haproxyConfig.go deleted file mode 100644 index 921245c6f..000000000 --- a/cmd/kk/pkg/loadbalancer/templates/haproxyConfig.go +++ /dev/null @@ -1,83 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "strconv" - "text/template" - - "github.com/lithammer/dedent" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -var HaproxyConfig = template.Must(template.New("haproxy.cfg").Parse( - dedent.Dedent(` -global - maxconn 4000 - log 127.0.0.1 local0 - -defaults - mode http - log global - option httplog - option dontlognull - option http-server-close - option redispatch - retries 5 - timeout http-request 5m - timeout queue 5m - timeout connect 30s - timeout client 30s - timeout server 15m - timeout http-keep-alive 30s - timeout check 30s - maxconn 4000 - -frontend healthz - bind *:{{ .LoadbalancerApiserverHealthcheckPort }} - mode http - monitor-uri /healthz - -frontend kube_api_frontend - bind 127.0.0.1:{{ .LoadbalancerApiserverPort }} - mode tcp - option tcplog - default_backend kube_api_backend - -backend kube_api_backend - mode tcp - balance leastconn - default-server inter 15s downinter 15s rise 2 fall 2 slowstart 60s maxconn 1000 maxqueue 256 weight 100 - {{- if ne .KubernetesType "k3s"}} - option httpchk GET /healthz - {{- end }} - http-check expect status 200 - {{- range .MasterNodes }} - server {{ . }} check check-ssl verify none - {{- end }} -`))) - -func MasterNodeStr(runtime connector.ModuleRuntime, conf *common.KubeConf) []string { - masterNodes := make([]string, len(runtime.GetHostsByRole(common.Master))) - for i, node := range runtime.GetHostsByRole(common.Master) { - masterNodes[i] = node.GetName() + " " + node.GetAddress() + ":" + strconv.Itoa(kubekeyapiv1alpha2.DefaultApiserverPort) - } - return masterNodes -} diff --git a/cmd/kk/pkg/loadbalancer/templates/haproxyManifest.go b/cmd/kk/pkg/loadbalancer/templates/haproxyManifest.go deleted file mode 100644 index d4f5f7151..000000000 --- a/cmd/kk/pkg/loadbalancer/templates/haproxyManifest.go +++ /dev/null @@ -1,66 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var HaproxyManifest = template.Must(template.New("haproxy.yaml").Parse( - dedent.Dedent(` -apiVersion: v1 -kind: Pod -metadata: - name: haproxy - namespace: kube-system - labels: - addonmanager.kubernetes.io/mode: Reconcile - k8s-app: kube-haproxy - annotations: - cfg-checksum: "{{ .Checksum }}" -spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - nodeSelector: - kubernetes.io/os: linux - priorityClassName: system-node-critical - containers: - - name: haproxy - image: {{ .HaproxyImage }} - imagePullPolicy: IfNotPresent - resources: - requests: - cpu: 25m - memory: 32M - livenessProbe: - httpGet: - path: /healthz - port: {{ .HealthCheckPort }} - readinessProbe: - httpGet: - path: /healthz - port: {{ .HealthCheckPort }} - volumeMounts: - - mountPath: /usr/local/etc/haproxy/ - name: etc-haproxy - readOnly: true - volumes: - - name: etc-haproxy - hostPath: - path: /etc/kubekey/haproxy -`))) diff --git a/cmd/kk/pkg/loadbalancer/templates/k3sKubevipManifest.go b/cmd/kk/pkg/loadbalancer/templates/k3sKubevipManifest.go deleted file mode 100644 index 827d4af44..000000000 --- a/cmd/kk/pkg/loadbalancer/templates/k3sKubevipManifest.go +++ /dev/null @@ -1,279 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -var K3sKubevipManifest = template.Must(template.New("kube-vip-rbac.yaml").Parse( - dedent.Dedent(`{{ if .BGPMode }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kube-vip - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - name: system:kube-vip-role -rules: - - apiGroups: [""] - resources: ["services", "services/status", "nodes", "endpoints"] - verbs: ["list","get","watch", "update"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["list", "get", "watch", "update", "create"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: system:kube-vip-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:kube-vip-role -subjects: -- kind: ServiceAccount - name: kube-vip - namespace: kube-system ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - creationTimestamp: null - labels: - app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: {{ .KubeVipVersion }} - name: kube-vip-ds - namespace: kube-system -spec: - selector: - matchLabels: - app.kubernetes.io/name: kube-vip-ds - template: - metadata: - creationTimestamp: null - labels: - app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: {{ .KubeVipVersion }} - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - containers: - - args: - - manager - env: - - name: vip_arp - value: "false" - - name: port - value: "6443" - - name: vip_interface - value: {{ .VipInterface }} - - name: vip_cidr - value: "32" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: vip_ddns - value: "false" - - name: svc_enable - value: "true" - - name: bgp_enable - value: "true" - - name: bgp_routerid - value: {{ .BGPRouterID }} - - name: bgp_as - value: "65000" - - name: bgp_peeraddress - - name: bgp_peerpass - - name: bgp_peeras - value: "65000" - - name: bgp_peers - value: {{ .BGPPeers }} - - name: lb_enable - value: "true" - - name: lb_port - value: "6443" - - name: lb_fwdmethod - value: local - - name: address - value: {{ .KubeVip }} - - name: prometheus_server - value: :2112 - image: {{ .KubevipImage }} - imagePullPolicy: IfNotPresent - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - hostNetwork: true - serviceAccountName: kube-vip - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - updateStrategy: {} -status: - currentNumberScheduled: 0 - desiredNumberScheduled: 0 - numberMisscheduled: 0 - numberReady: 0 -{{ else }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kube-vip - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - name: system:kube-vip-role -rules: - - apiGroups: [""] - resources: ["services", "services/status", "nodes", "endpoints"] - verbs: ["list","get","watch", "update"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["list", "get", "watch", "update", "create"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: system:kube-vip-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:kube-vip-role -subjects: -- kind: ServiceAccount - name: kube-vip - namespace: kube-system ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - creationTimestamp: null - labels: - app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: {{ .KubeVipVersion }} - name: kube-vip-ds - namespace: kube-system -spec: - selector: - matchLabels: - app.kubernetes.io/name: kube-vip-ds - template: - metadata: - creationTimestamp: null - labels: - app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: {{ .KubeVipVersion }} - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - containers: - - args: - - manager - env: - - name: vip_arp - value: "true" - - name: port - value: "6443" - - name: vip_interface - value: {{ .VipInterface }} - - name: vip_cidr - value: "32" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: vip_ddns - value: "false" - - name: svc_enable - value: "true" - - name: vip_leaderelection - value: "true" - - name: vip_leaseduration - value: "5" - - name: vip_renewdeadline - value: "3" - - name: vip_retryperiod - value: "1" - - name: lb_enable - value: "true" - - name: lb_port - value: "6443" - - name: lb_fwdmethod - value: local - - name: address - value: {{ .KubeVip }} - - name: prometheus_server - value: :2112 - image: {{ .KubevipImage }} - imagePullPolicy: IfNotPresent - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - hostNetwork: true - serviceAccountName: kube-vip - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - updateStrategy: {} -status: - currentNumberScheduled: 0 - desiredNumberScheduled: 0 - numberMisscheduled: 0 - numberReady: 0 -{{ end }} -`))) diff --git a/cmd/kk/pkg/loadbalancer/templates/kubevipManifest.go b/cmd/kk/pkg/loadbalancer/templates/kubevipManifest.go deleted file mode 100644 index 111e418c8..000000000 --- a/cmd/kk/pkg/loadbalancer/templates/kubevipManifest.go +++ /dev/null @@ -1,164 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -var KubevipManifest = template.Must(template.New("kube-vip.yaml").Parse( - dedent.Dedent(`{{ if .BGPMode }} -apiVersion: v1 -kind: Pod -metadata: - creationTimestamp: null - name: kube-vip - namespace: kube-system -spec: - containers: - - args: - - manager - env: - - name: vip_arp - value: "false" - - name: port - value: "6443" - - name: vip_interface - value: {{ .VipInterface }} - - name: vip_cidr - value: "32" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: vip_ddns - value: "false" - - name: svc_enable - value: "true" - - name: bgp_enable - value: "true" - - name: bgp_routerid - value: {{ .BGPRouterID }} - - name: bgp_as - value: "65000" - - name: bgp_peeraddress - - name: bgp_peerpass - - name: bgp_peeras - value: "65000" - - name: bgp_peers - value: {{ .BGPPeers }} - - name: lb_enable - value: "true" - - name: lb_port - value: "6443" - - name: lb_fwdmethod - value: local - - name: address - value: {{ .KubeVip }} - - name: prometheus_server - value: :2112 - image: {{ .KubevipImage }} - imagePullPolicy: IfNotPresent - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - volumeMounts: - - mountPath: /etc/kubernetes/admin.conf - name: kubeconfig - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 - hostNetwork: true - volumes: - - hostPath: - path: /etc/kubernetes/admin.conf - name: kubeconfig -status: {} -{{ else }} -apiVersion: v1 -kind: Pod -metadata: - creationTimestamp: null - name: kube-vip - namespace: kube-system -spec: - containers: - - args: - - manager - env: - - name: vip_arp - value: "true" - - name: port - value: "6443" - - name: vip_interface - value: {{ .VipInterface }} - - name: vip_cidr - value: "32" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: vip_ddns - value: "false" - - name: svc_enable - value: "true" - - name: vip_leaderelection - value: "true" - - name: vip_leaseduration - value: "5" - - name: vip_renewdeadline - value: "3" - - name: vip_retryperiod - value: "1" - - name: lb_enable - value: "true" - - name: lb_port - value: "6443" - - name: address - value: {{ .KubeVip }} - image: {{ .KubevipImage }} - imagePullPolicy: IfNotPresent - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - - SYS_TIME - volumeMounts: - - mountPath: /etc/kubernetes/admin.conf - name: kubeconfig - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 - hostNetwork: true - volumes: - - hostPath: - path: /etc/kubernetes/admin.conf - name: kubeconfig -status: {} -{{ end }} -`))) diff --git a/cmd/kk/pkg/phase/artifact/artifact.go b/cmd/kk/pkg/phase/artifact/artifact.go deleted file mode 100755 index 5d540ca83..000000000 --- a/cmd/kk/pkg/phase/artifact/artifact.go +++ /dev/null @@ -1,64 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package artifact - -import ( - "errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/artifact" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" -) - -func NewArtifactImportPipeline(runtime *common.KubeRuntime) error { - - m := []module.Module{ - &artifact.UnArchiveModule{}, - } - - p := pipeline.Pipeline{ - Name: "ArtifactImportPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func ArtifactImport(args common.Argument) error { - var loaderType string - - loaderType = common.AllInOne - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewArtifactImportPipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/binary/create_binary.go b/cmd/kk/pkg/phase/binary/create_binary.go deleted file mode 100755 index b29987e15..000000000 --- a/cmd/kk/pkg/phase/binary/create_binary.go +++ /dev/null @@ -1,78 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package binary - -import ( - "errors" - "fmt" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/binaries" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" -) - -func NewCreateBinaryPipeline(runtime *common.KubeRuntime) error { - - m := []module.Module{ - &precheck.NodePreCheckModule{}, - &binaries.NodeBinariesModule{}, - &SyncBinaryModule{}, - } - - p := pipeline.Pipeline{ - Name: "CreateBinaryPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func CreateBinary(args common.Argument, downloadCmd string) error { - args.DownloadCommand = func(path, url string) string { - // this is an extension point for downloading tools, for example users can set the timeout, proxy or retry under - // some poor network environment. Or users even can choose another cli, it might be wget. - // perhaps we should have a build-in download function instead of totally rely on the external one - return fmt.Sprintf(downloadCmd, path, url) - } - var loaderType string - - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewCreateBinaryPipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/binary/modules.go b/cmd/kk/pkg/phase/binary/modules.go deleted file mode 100755 index 40208c8f9..000000000 --- a/cmd/kk/pkg/phase/binary/modules.go +++ /dev/null @@ -1,45 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package binary - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" -) - -type SyncBinaryModule struct { - common.KubeModule -} - -func (p *SyncBinaryModule) Init() { - p.Name = "SyncBinaryModule" - p.Desc = "synchronize kubernetes binaries" - - syncBinary := &task.RemoteTask{ - Name: "SyncKubeBinary", - Desc: "Synchronize kubernetes binaries", - Hosts: p.Runtime.GetHostsByRole(common.K8s), - Action: new(kubernetes.SyncKubeBinary), - Parallel: true, - Retry: 2, - } - - p.Tasks = []task.Interface{ - syncBinary, - } -} diff --git a/cmd/kk/pkg/phase/binary/task.go b/cmd/kk/pkg/phase/binary/task.go deleted file mode 100755 index 4300c6598..000000000 --- a/cmd/kk/pkg/phase/binary/task.go +++ /dev/null @@ -1,67 +0,0 @@ -package binary - -import ( - "fmt" - - "github.com/pkg/errors" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" -) - -type GetBinaryPath struct { - common.KubeAction - Binaries []string -} - -func (g *GetBinaryPath) Execute(runtime connector.Runtime) error { - cfg := g.KubeConf.Cluster - - archMap := make(map[string]bool) - for _, host := range cfg.Hosts { - switch host.Arch { - case "amd64": - archMap["amd64"] = true - case "arm64": - archMap["arm64"] = true - default: - return errors.New(fmt.Sprintf("Unsupported architecture: %s", host.Arch)) - } - } - - for arch := range archMap { - if err := setBinaryPath(g.KubeConf, runtime.GetWorkDir(), arch, g.Binaries, g.PipelineCache); err != nil { - return err - } - } - return nil -} - -func setBinaryPath(kubeConf *common.KubeConf, path, arch string, binaries []string, pipelineCache *cache.Cache) error { - binariesMap := make(map[string]*files.KubeBinary) - var kubeBinary *files.KubeBinary - for _, binary := range binaries { - switch binary { - case "etcd": - kubeBinary = files.NewKubeBinary(binary, arch, kubekeyapiv1alpha2.DefaultEtcdVersion, path, kubeConf.Arg.DownloadCommand) - case "docker": - kubeBinary = files.NewKubeBinary(binary, arch, kubekeyapiv1alpha2.DefaultDockerVersion, path, kubeConf.Arg.DownloadCommand) - case "containerd": - kubeBinary = files.NewKubeBinary(binary, arch, kubekeyapiv1alpha2.DefaultContainerdVersion, path, kubeConf.Arg.DownloadCommand) - case "helm": - kubeBinary = files.NewKubeBinary(binary, arch, kubekeyapiv1alpha2.DefaultHelmVersion, path, kubeConf.Arg.DownloadCommand) - case "crictl": - kubeBinary = files.NewKubeBinary("crictl", arch, kubekeyapiv1alpha2.DefaultCrictlVersion, path, kubeConf.Arg.DownloadCommand) - case "runc": - kubeBinary = files.NewKubeBinary("runc", arch, kubekeyapiv1alpha2.DefaultRuncVersion, path, kubeConf.Arg.DownloadCommand) - default: - return errors.New(fmt.Sprintf("Unsupported binary name: %s", binary)) - } - binariesMap[kubeBinary.ID] = kubeBinary - } - pipelineCache.Set(common.KubeBinaries+"-"+arch, binariesMap) - return nil -} diff --git a/cmd/kk/pkg/phase/binary/upgrade_binary.go b/cmd/kk/pkg/phase/binary/upgrade_binary.go deleted file mode 100755 index e1639ed6f..000000000 --- a/cmd/kk/pkg/phase/binary/upgrade_binary.go +++ /dev/null @@ -1,78 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package binary - -import ( - "errors" - "fmt" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/binaries" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/precheck" -) - -func NewUpgradeBinaryPipeline(runtime *common.KubeRuntime) error { - - m := []module.Module{ - &precheck.UpgradePreCheckModule{}, - &binaries.NodeBinariesModule{}, - &SyncBinaryModule{}, - } - - p := pipeline.Pipeline{ - Name: "UpgradeBinaryPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func UpgradeBinary(args common.Argument, downloadCmd string) error { - args.DownloadCommand = func(path, url string) string { - // this is an extension point for downloading tools, for example users can set the timeout, proxy or retry under - // some poor network environment. Or users even can choose another cli, it might be wget. - // perhaps we should have a build-in download function instead of totally rely on the external one - return fmt.Sprintf(downloadCmd, path, url) - } - var loaderType string - - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewUpgradeBinaryPipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/confirm/confirm.go b/cmd/kk/pkg/phase/confirm/confirm.go deleted file mode 100644 index fe847d3e6..000000000 --- a/cmd/kk/pkg/phase/confirm/confirm.go +++ /dev/null @@ -1,82 +0,0 @@ -package confirm - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" -) - -type UpgradeK8sConfirmModule struct { - common.KubeModule -} - -func (u *UpgradeK8sConfirmModule) Init() { - u.Name = "UpgradeKsConfirmModule" - u.Desc = "Display upgrade kubesphere confirmation form" - - display := &task.LocalTask{ - Name: "ConfirmForm", - Desc: "Display confirmation form", - Action: new(UpgradeK8sConfirm), - } - - u.Tasks = []task.Interface{ - display, - } -} - -type UpgradeKsConfirmModule struct { - common.KubeModule -} - -func (u *UpgradeKsConfirmModule) Init() { - u.Name = "UpgradeKsConfirmModule" - u.Desc = "Display upgrade kubesphere confirmation form" - - display := &task.LocalTask{ - Name: "ConfirmForm", - Desc: "Display confirmation form", - Action: new(UpgradeKsConfirm), - } - - u.Tasks = []task.Interface{ - display, - } -} - -type CreateK8sConfirmModule struct { - common.KubeModule -} - -func (u *CreateK8sConfirmModule) Init() { - u.Name = "CreateKsConfirmModule" - u.Desc = "Display Create kubesphere confirmation form" - - display := &task.LocalTask{ - Name: "ConfirmForm", - Desc: "Display confirmation form", - Action: new(CreateK8sConfirm), - } - - u.Tasks = []task.Interface{ - display, - } -} - -type CreateKsConfirmModule struct { - common.KubeModule -} - -func (u *CreateKsConfirmModule) Init() { - u.Name = "CreateKsConfirmModule" - u.Desc = "Display Create kubesphere confirmation form" - - display := &task.LocalTask{ - Name: "ConfirmForm", - Desc: "Display confirmation form", - Action: new(CreateKsConfirm), - } - - u.Tasks = []task.Interface{ - display, - } -} diff --git a/cmd/kk/pkg/phase/confirm/task.go b/cmd/kk/pkg/phase/confirm/task.go deleted file mode 100644 index 365e31aef..000000000 --- a/cmd/kk/pkg/phase/confirm/task.go +++ /dev/null @@ -1,302 +0,0 @@ -package confirm - -import ( - "bufio" - "errors" - "fmt" - "os" - "strings" - - "github.com/mitchellh/mapstructure" - "github.com/modood/table" - versionutil "k8s.io/apimachinery/pkg/util/version" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/confirm" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" -) - -type UpgradeK8sConfirm struct { - common.KubeAction -} - -func (u *UpgradeK8sConfirm) Execute(runtime connector.Runtime) error { - pre := make([]map[string]string, len(runtime.GetAllHosts()), len(runtime.GetAllHosts())) - for i, host := range runtime.GetAllHosts() { - if v, ok := host.GetCache().Get(common.NodePreCheck); ok { - pre[i] = v.(map[string]string) - } else { - return errors.New("get node check result failed by host cache") - } - } - - results := make([]confirm.PreCheckResults, len(pre), len(pre)) - for i := range pre { - var result confirm.PreCheckResults - _ = mapstructure.Decode(pre[i], &result) - results[i] = result - } - table.OutputA(results) - fmt.Println() - - warningFlag := false - cmp, err := versionutil.MustParseSemantic(u.KubeConf.Cluster.Kubernetes.Version).Compare("v1.19.0") - if err != nil { - logger.Log.Fatalf("Failed to compare kubernetes version: %v", err) - } - if cmp == 0 || cmp == 1 { - for _, result := range results { - if len(result.Docker) != 0 { - dockerVersion, err := confirm.RefineDockerVersion(result.Docker) - if err != nil { - logger.Log.Fatalf("Failed to get docker version: %v", err) - } - cmp, err := versionutil.MustParseSemantic(dockerVersion).Compare("20.10.0") - if err != nil { - logger.Log.Fatalf("Failed to compare docker version: %v", err) - } - warningFlag = warningFlag || (cmp == -1) - } - } - if warningFlag { - fmt.Println(` -Warning: - - An old Docker version may cause the failure of upgrade. It is recommended that you upgrade Docker to 20.10+ beforehand. - - Issue: https://github.com/kubernetes/kubernetes/issues/101056`) - fmt.Print("\n") - } - } - - nodeStats, ok := u.PipelineCache.GetMustString(common.ClusterNodeStatus) - if !ok { - return errors.New("get cluster nodes status failed by pipeline cache") - } - fmt.Println("Cluster nodes status:") - fmt.Println(nodeStats + "\n") - - fmt.Println("Upgrade Confirmation:") - currentK8sVersion, ok := u.PipelineCache.GetMustString(common.K8sVersion) - if !ok { - return errors.New("get current Kubernetes version failed by pipeline cache") - } - fmt.Printf("kubernetes version: %s to %s\n", currentK8sVersion, u.KubeConf.Cluster.Kubernetes.Version) - - fmt.Println() - - if k8sVersion, err := versionutil.ParseGeneric(u.KubeConf.Cluster.Kubernetes.Version); err == nil { - if cri, ok := u.PipelineCache.GetMustString(common.ClusterNodeCRIRuntimes); ok { - k8sV124 := versionutil.MustParseSemantic("v1.24.0") - if k8sVersion.AtLeast(k8sV124) && versionutil.MustParseSemantic(currentK8sVersion).LessThan(k8sV124) && strings.Contains(cri, "docker") { - fmt.Println("[Notice]") - fmt.Println("Pre-upgrade check failed. The container runtime of the current cluster is Docker.") - fmt.Println("Kubernetes v1.24 and later no longer support dockershim and Docker.") - fmt.Println("Make sure you have completed the migration from Docker to other container runtimes that are compatible with the Kubernetes CRI.") - fmt.Println("For more information, see:") - fmt.Println("https://kubernetes.io/docs/setup/production-environment/container-runtimes/#container-runtimes") - fmt.Println("https://kubernetes.io/blog/2022/02/17/dockershim-faq/") - fmt.Println("") - } - } - } - - reader := bufio.NewReader(os.Stdin) - confirmOK := false - for !confirmOK { - fmt.Printf("Continue upgrading kubernetes? [yes/no]: ") - input, err := reader.ReadString('\n') - if err != nil { - return err - } - input = strings.ToLower(strings.TrimSpace(input)) - - switch input { - case "yes", "y": - confirmOK = true - case "no", "n": - os.Exit(0) - default: - continue - } - } - return nil -} - -type UpgradeKsConfirm struct { - common.KubeAction -} - -func (u *UpgradeKsConfirm) Execute(runtime connector.Runtime) error { - pre := make([]map[string]string, len(runtime.GetAllHosts()), len(runtime.GetAllHosts())) - for i, host := range runtime.GetAllHosts() { - if v, ok := host.GetCache().Get(common.NodePreCheck); ok { - pre[i] = v.(map[string]string) - } else { - return errors.New("get node check result failed by host cache") - } - } - - if u.KubeConf.Cluster.KubeSphere.Enabled { - currentKsVersion, ok := u.PipelineCache.GetMustString(common.KubeSphereVersion) - if !ok { - return errors.New("get current KubeSphere version failed by pipeline cache") - } - fmt.Printf("kubesphere version: %s to %s\n", currentKsVersion, u.KubeConf.Cluster.KubeSphere.Version) - } - fmt.Println() - - reader := bufio.NewReader(os.Stdin) - confirmOK := false - for !confirmOK { - fmt.Printf("Continue upgrading KubeSphere? [yes/no]: ") - input, err := reader.ReadString('\n') - if err != nil { - return err - } - input = strings.ToLower(strings.TrimSpace(input)) - - switch input { - case "yes", "y": - confirmOK = true - case "no", "n": - os.Exit(0) - default: - continue - } - } - return nil -} - -type CreateK8sConfirm struct { - common.KubeAction -} - -func (c *CreateK8sConfirm) Execute(runtime connector.Runtime) error { - var ( - results []confirm.PreCheckResults - stopFlag bool - ) - - pre := make([]map[string]string, 0, len(runtime.GetAllHosts())) - for _, host := range runtime.GetAllHosts() { - if v, ok := host.GetCache().Get(common.NodePreCheck); ok { - pre = append(pre, v.(map[string]string)) - } else { - return errors.New("get node check result failed by host cache") - } - } - - for node := range pre { - var result confirm.PreCheckResults - _ = mapstructure.Decode(pre[node], &result) - results = append(results, result) - } - table.OutputA(results) - reader := bufio.NewReader(os.Stdin) - - if c.KubeConf.Arg.Artifact == "" { - for _, host := range results { - if host.Sudo == "" { - logger.Log.Errorf("%s: sudo is required.", host.Name) - stopFlag = true - } - - if host.Conntrack == "" { - logger.Log.Errorf("%s: conntrack is required.", host.Name) - stopFlag = true - } - - if host.Socat == "" { - logger.Log.Errorf("%s: socat is required.", host.Name) - stopFlag = true - } - } - } - - fmt.Println("") - fmt.Println("This is a simple check of your environment.") - fmt.Println("Before installation, ensure that your machines meet all requirements specified at") - fmt.Println("https://github.com/kubesphere/kubekey#requirements-and-recommendations") - fmt.Println("") - - if k8sVersion, err := versionutil.ParseGeneric(c.KubeConf.Cluster.Kubernetes.Version); err == nil { - if k8sVersion.AtLeast(versionutil.MustParseSemantic("v1.24.0")) && c.KubeConf.Cluster.Kubernetes.ContainerManager == common.Docker { - fmt.Println("[Notice]") - fmt.Println("Incorrect runtime. Please specify a container runtime other than Docker to install Kubernetes v1.24 or later.") - fmt.Println("You can set \"spec.kubernetes.containerManager\" in the configuration file to \"containerd\" or add \"--container-manager containerd\" to the \"./kk create cluster\" command.") - fmt.Println("For more information, see:") - fmt.Println("https://github.com/kubesphere/kubekey/blob/master/docs/commands/kk-create-cluster.md") - fmt.Println("https://kubernetes.io/docs/setup/production-environment/container-runtimes/#container-runtimes") - fmt.Println("https://kubernetes.io/blog/2022/02/17/dockershim-faq/") - fmt.Println("") - stopFlag = true - } - } - - if stopFlag { - os.Exit(1) - } - - confirmOK := false - for !confirmOK { - fmt.Printf("Continue this init the cluster? [yes/no]: ") - input, err := reader.ReadString('\n') - if err != nil { - logger.Log.Fatal(err) - } - input = strings.TrimSpace(strings.ToLower(input)) - - switch strings.ToLower(input) { - case "yes", "y": - confirmOK = true - case "no", "n": - os.Exit(0) - default: - continue - } - } - return nil -} - -type CreateKsConfirm struct { - common.KubeAction -} - -func (u *CreateKsConfirm) Execute(runtime connector.Runtime) error { - pre := make([]map[string]string, len(runtime.GetAllHosts()), len(runtime.GetAllHosts())) - for i, host := range runtime.GetAllHosts() { - if v, ok := host.GetCache().Get(common.NodePreCheck); ok { - pre[i] = v.(map[string]string) - } else { - return errors.New("get node check result failed by host cache") - } - } - - if u.KubeConf.Cluster.KubeSphere.Enabled { - fmt.Printf("desired kubesphere version: %s\n", u.KubeConf.Cluster.KubeSphere.Version) - } - fmt.Println() - - reader := bufio.NewReader(os.Stdin) - confirmOK := false - for !confirmOK { - fmt.Printf("Continue install KubeSphere? [yes/no]: ") - input, err := reader.ReadString('\n') - if err != nil { - return err - } - input = strings.ToLower(strings.TrimSpace(input)) - - switch input { - case "yes", "y": - confirmOK = true - case "no", "n": - os.Exit(0) - default: - continue - } - } - return nil -} diff --git a/cmd/kk/pkg/phase/etcd/etcd.go b/cmd/kk/pkg/phase/etcd/etcd.go deleted file mode 100755 index ce40781f3..000000000 --- a/cmd/kk/pkg/phase/etcd/etcd.go +++ /dev/null @@ -1,72 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package etcd - -import ( - "errors" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/etcd" -) - -func NewCreateEtcdPipeline(runtime *common.KubeRuntime) error { - m := []module.Module{ - &PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.CertsModule{}, - &etcd.InstallETCDBinaryModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.ConfigureModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.BackupModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - } - - p := pipeline.Pipeline{ - Name: "CreateEtcdPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func CreateEtcd(args common.Argument) error { - var loaderType string - - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewCreateEtcdPipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/etcd/modules.go b/cmd/kk/pkg/phase/etcd/modules.go deleted file mode 100755 index 7e5ca11fc..000000000 --- a/cmd/kk/pkg/phase/etcd/modules.go +++ /dev/null @@ -1,42 +0,0 @@ -package etcd - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/etcd" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/binary" -) - -type PreCheckModule struct { - common.KubeModule - Skip bool -} - -func (p *PreCheckModule) IsSkip() bool { - return p.Skip -} - -func (p *PreCheckModule) Init() { - p.Name = "ETCDPreCheckModule" - p.Desc = "Get ETCD cluster status" - - getStatus := &task.RemoteTask{ - Name: "GetETCDStatus", - Desc: "Get etcd status", - Hosts: p.Runtime.GetHostsByRole(common.ETCD), - Action: new(etcd.GetStatus), - Parallel: false, - Retry: 0, - } - - setBinaryCache := &task.LocalTask{ - Name: "SetEtcdBinaryCache", - Desc: "Set Etcd Binary Path in PipelineCache", - Action: &binary.GetBinaryPath{Binaries: []string{"etcd"}}, - } - - p.Tasks = []task.Interface{ - getStatus, - setBinaryCache, - } -} diff --git a/cmd/kk/pkg/phase/images/create_images.go b/cmd/kk/pkg/phase/images/create_images.go deleted file mode 100755 index 6bbb0f913..000000000 --- a/cmd/kk/pkg/phase/images/create_images.go +++ /dev/null @@ -1,72 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package images - -import ( - "errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" -) - -func NewCreateImagesPipeline(runtime *common.KubeRuntime) error { - m := []module.Module{ - &precheck.NodePreCheckModule{}, - &setBinaryCacheModule{}, - &kubernetes.StatusModule{}, - &container.InstallContainerModule{}, - } - - p := pipeline.Pipeline{ - Name: "CreateImagesPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func CreateImages(args common.Argument) error { - var loaderType string - - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewCreateImagesPipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/images/modules.go b/cmd/kk/pkg/phase/images/modules.go deleted file mode 100644 index 4fe937481..000000000 --- a/cmd/kk/pkg/phase/images/modules.go +++ /dev/null @@ -1,66 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package images - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/binary" -) - -type UpgradeImagesModule struct { - common.KubeModule -} - -func (p *UpgradeImagesModule) Init() { - p.Name = "UpgradeImagesModule" - p.Desc = "pull the images that cluster need" - - pull := &task.RemoteTask{ - Name: "PullImages", - Desc: "Start to pull images on all nodes", - Hosts: p.Runtime.GetHostsByRole(common.K8s), - Prepare: new(kubernetes.NotEqualPlanVersion), - Action: new(images.PullImage), - Parallel: true, - } - - p.Tasks = []task.Interface{ - pull, - } -} - -type setBinaryCacheModule struct { - common.KubeModule -} - -func (p *setBinaryCacheModule) Init() { - p.Name = "setBinaryCacheModule" - p.Desc = "set the docker and containerd binary paths in cache" - - setBinaryCache := &task.LocalTask{ - Name: "SetBinaryCache", - Desc: "Set Binary Path in PipelineCache", - Action: &binary.GetBinaryPath{Binaries: []string{"docker", "containerd", "runc", "crictl"}}, - } - - p.Tasks = []task.Interface{ - setBinaryCache, - } -} diff --git a/cmd/kk/pkg/phase/images/upgrade_images.go b/cmd/kk/pkg/phase/images/upgrade_images.go deleted file mode 100755 index 65016684e..000000000 --- a/cmd/kk/pkg/phase/images/upgrade_images.go +++ /dev/null @@ -1,69 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package images - -import ( - "errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/precheck" -) - -func NewUpgradeImagesPipeline(runtime *common.KubeRuntime) error { - - m := []module.Module{ - &precheck.UpgradePreCheckModule{}, - &UpgradeImagesModule{}, - } - - p := pipeline.Pipeline{ - Name: "UpgradeImagesPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func UpgradeImages(args common.Argument) error { - var loaderType string - - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewUpgradeImagesPipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/kubernetes/configure.go b/cmd/kk/pkg/phase/kubernetes/configure.go deleted file mode 100755 index 82e82bf99..000000000 --- a/cmd/kk/pkg/phase/kubernetes/configure.go +++ /dev/null @@ -1,89 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubernetes - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/addons" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/certs" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/filesystem" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/network" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/storage" -) - -func NewCreateConfigureKubernetesPipeline(runtime *common.KubeRuntime) error { - skipLocalStorage := true - if runtime.Arg.DeployLocalStorage != nil { - skipLocalStorage = !*runtime.Arg.DeployLocalStorage - } else if runtime.Cluster.KubeSphere.Enabled { - skipLocalStorage = false - } - m := []module.Module{ - &precheck.NodePreCheckModule{}, - &kubernetes.StatusModule{}, - &network.DeployNetworkPluginModule{}, - &kubernetes.ConfigureKubernetesModule{}, - &filesystem.ChownModule{}, - &certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()}, - &kubernetes.SaveKubeConfigModule{}, - &plugins.DeployPluginsModule{}, - &addons.AddonsModule{}, - &storage.DeployLocalVolumeModule{Skip: skipLocalStorage}, - } - - p := pipeline.Pipeline{ - Name: "CreateConfigureKubernetesPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func CreateConfigureKubernetes(args common.Argument) error { - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewCreateConfigureKubernetesPipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/kubernetes/init.go b/cmd/kk/pkg/phase/kubernetes/init.go deleted file mode 100644 index 9d11bf237..000000000 --- a/cmd/kk/pkg/phase/kubernetes/init.go +++ /dev/null @@ -1,76 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubernetes - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/confirm" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/dns" -) - -func NewCreateInitClusterPipeline(runtime *common.KubeRuntime) error { - m := []module.Module{ - &precheck.NodePreCheckModule{}, - &kubernetes.StatusModule{}, - &confirm.CreateK8sConfirmModule{}, - &InstallKubeletModule{}, - &kubernetes.InitKubernetesModule{}, - &dns.ClusterDNSModule{}, - } - - p := pipeline.Pipeline{ - Name: "CreateInitClusterPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func CreateInitCluster(args common.Argument) error { - - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewCreateInitClusterPipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/kubernetes/join.go b/cmd/kk/pkg/phase/kubernetes/join.go deleted file mode 100644 index 75c1bfb63..000000000 --- a/cmd/kk/pkg/phase/kubernetes/join.go +++ /dev/null @@ -1,70 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubernetes - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" -) - -func NewCreateJoinNodesPipeline(runtime *common.KubeRuntime) error { - m := []module.Module{ - &precheck.NodePreCheckModule{}, - &kubernetes.StatusModule{}, - &kubernetes.JoinNodesModule{}, - } - - p := pipeline.Pipeline{ - Name: "CreateJoinNodesPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func CreateJoinNodes(args common.Argument) error { - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewCreateJoinNodesPipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/kubernetes/modules.go b/cmd/kk/pkg/phase/kubernetes/modules.go deleted file mode 100644 index 4700d2f3c..000000000 --- a/cmd/kk/pkg/phase/kubernetes/modules.go +++ /dev/null @@ -1,70 +0,0 @@ -package kubernetes - -import ( - "path/filepath" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes/templates" -) - -type InstallKubeletModule struct { - common.KubeModule -} - -func (i *InstallKubeletModule) Init() { - i.Name = "InstallKubeletModule" - i.Desc = "Install kubernetes cluster" - - chmodKubelet := &task.RemoteTask{ - Name: "ChmodKubelet", - Desc: "Change kubelet mode", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &kubernetes.NodeInCluster{Not: true}, - Action: new(kubernetes.ChmodKubelet), - Parallel: true, - Retry: 2, - } - - generateKubeletService := &task.RemoteTask{ - Name: "GenerateKubeletService", - Desc: "Generate kubelet service", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &kubernetes.NodeInCluster{Not: true}, - Action: &action.Template{ - Template: templates.KubeletService, - Dst: filepath.Join("/etc/systemd/system/", templates.KubeletService.Name()), - }, - Parallel: true, - Retry: 2, - } - - enableKubelet := &task.RemoteTask{ - Name: "EnableKubelet", - Desc: "Enable kubelet service", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &kubernetes.NodeInCluster{Not: true}, - Action: new(kubernetes.EnableKubelet), - Parallel: true, - Retry: 5, - } - - generateKubeletEnv := &task.RemoteTask{ - Name: "GenerateKubeletEnv", - Desc: "Generate kubelet env", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &kubernetes.NodeInCluster{Not: true}, - Action: new(kubernetes.GenerateKubeletEnv), - Parallel: true, - Retry: 2, - } - - i.Tasks = []task.Interface{ - chmodKubelet, - generateKubeletService, - enableKubelet, - generateKubeletEnv, - } -} diff --git a/cmd/kk/pkg/phase/kubesphere/create_kubesphere.go b/cmd/kk/pkg/phase/kubesphere/create_kubesphere.go deleted file mode 100755 index 76dd62862..000000000 --- a/cmd/kk/pkg/phase/kubesphere/create_kubesphere.go +++ /dev/null @@ -1,74 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package alpha - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubesphere" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/confirm" -) - -func NewCreateKubeSpherePipeline(runtime *common.KubeRuntime) error { - - m := []module.Module{ - &precheck.NodePreCheckModule{}, - &confirm.CreateKsConfirmModule{}, - &kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &kubesphere.CheckResultModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - } - - p := pipeline.Pipeline{ - Name: "CreateKubeSpherePipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func CreateKubeSphere(args common.Argument) error { - - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewCreateKubeSpherePipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/kubesphere/upgrade_kubesphere.go b/cmd/kk/pkg/phase/kubesphere/upgrade_kubesphere.go deleted file mode 100755 index 8eebf1903..000000000 --- a/cmd/kk/pkg/phase/kubesphere/upgrade_kubesphere.go +++ /dev/null @@ -1,77 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package alpha - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubesphere" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/confirm" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/precheck" -) - -func NewUpgradeKubeSpherePipeline(runtime *common.KubeRuntime) error { - - m := []module.Module{ - &precheck.UpgradeKubeSpherePreCheckModule{}, - &precheck.UpgradeksPhaseDependencyCheckModule{}, - &confirm.UpgradeKsConfirmModule{}, - &kubesphere.CleanClusterConfigurationModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &kubesphere.ConvertModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &kubesphere.CheckResultModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - } - - p := pipeline.Pipeline{ - Name: "UpgradeKubeSpherePipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func UpgradeKubeSphere(args common.Argument) error { - - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewUpgradeKubeSpherePipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/nodes/modules.go b/cmd/kk/pkg/phase/nodes/modules.go deleted file mode 100644 index 905329267..000000000 --- a/cmd/kk/pkg/phase/nodes/modules.go +++ /dev/null @@ -1,72 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package nodes - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" -) - -type UpgradeNodesModule struct { - common.KubeModule -} - -func (p *UpgradeNodesModule) Init() { - p.Name = "UpgradeNodesModule" - p.Desc = "Upgrade cluster on all nodes" - - upgradeKubeMaster := &task.RemoteTask{ - Name: "UpgradeClusterOnMaster", - Desc: "Upgrade cluster on master", - Hosts: p.Runtime.GetHostsByRole(common.Master), - Prepare: new(kubernetes.NotEqualPlanVersion), - Action: &kubernetes.UpgradeKubeMaster{ModuleName: p.Name}, - Parallel: false, - } - - cluster := kubernetes.NewKubernetesStatus() - p.PipelineCache.GetOrSet(common.ClusterStatus, cluster) - - clusterStatus := &task.RemoteTask{ - Name: "GetClusterStatus", - Desc: "Get kubernetes cluster status", - Hosts: p.Runtime.GetHostsByRole(common.Master), - Prepare: new(kubernetes.NotEqualPlanVersion), - Action: new(kubernetes.GetClusterStatus), - Parallel: false, - } - - upgradeNodes := &task.RemoteTask{ - Name: "UpgradeClusterOnWorker", - Desc: "Upgrade cluster on worker", - Hosts: p.Runtime.GetHostsByRole(common.Worker), - Prepare: &prepare.PrepareCollection{ - new(kubernetes.NotEqualPlanVersion), - new(common.OnlyWorker), - }, - Action: &kubernetes.UpgradeKubeWorker{ModuleName: p.Name}, - Parallel: false, - } - - p.Tasks = []task.Interface{ - upgradeKubeMaster, - clusterStatus, - upgradeNodes, - } -} diff --git a/cmd/kk/pkg/phase/nodes/nodes.go b/cmd/kk/pkg/phase/nodes/nodes.go deleted file mode 100755 index ddd7f27f0..000000000 --- a/cmd/kk/pkg/phase/nodes/nodes.go +++ /dev/null @@ -1,71 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package nodes - -import ( - "errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/confirm" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/precheck" -) - -func NewUpgradeNodesPipeline(runtime *common.KubeRuntime) error { - - m := []module.Module{ - &precheck.UpgradePreCheckModule{}, - &confirm.UpgradeK8sConfirmModule{}, - &UpgradeNodesModule{}, - } - - p := pipeline.Pipeline{ - Name: "UpgradeNodesPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func UpgradeNodes(args common.Argument) error { - var loaderType string - - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewUpgradeNodesPipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/os/config.go b/cmd/kk/pkg/phase/os/config.go deleted file mode 100755 index 907d38aca..000000000 --- a/cmd/kk/pkg/phase/os/config.go +++ /dev/null @@ -1,71 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package os - -import ( - "errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" -) - -func NewConfigOSPipeline(runtime *common.KubeRuntime) error { - - m := []module.Module{ - &precheck.NodePreCheckModule{}, - &os.RepositoryModule{Skip: !runtime.Arg.InstallPackages}, - &os.ConfigureOSModule{Skip: runtime.Cluster.System.SkipConfigureOS}, - } - - p := pipeline.Pipeline{ - Name: "ConfigOSPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func ConfigOS(args common.Argument) error { - var loaderType string - - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewConfigOSPipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/precheck/modules.go b/cmd/kk/pkg/phase/precheck/modules.go deleted file mode 100755 index 1e75a5f59..000000000 --- a/cmd/kk/pkg/phase/precheck/modules.go +++ /dev/null @@ -1,186 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package precheck - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" -) - -type UpgradePreCheckModule struct { - common.KubeModule -} - -func (c *UpgradePreCheckModule) Init() { - c.Name = "UpgradePreCheckModule" - c.Desc = "Do pre-check on for upgrade phase" - - nodePreCheck := &task.RemoteTask{ - Name: "NodePreCheck", - Desc: "A pre-check on nodes", - Hosts: c.Runtime.GetAllHosts(), - Action: new(precheck.NodePreCheck), - Parallel: true, - } - - getKubeConfig := &task.RemoteTask{ - Name: "GetKubeConfig", - Desc: "Get KubeConfig file", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(precheck.GetKubeConfig), - Parallel: true, - } - - getAllNodesK8sVersion := &task.RemoteTask{ - Name: "GetAllNodesK8sVersion", - Desc: "Get all nodes Kubernetes version", - Hosts: c.Runtime.GetHostsByRole(common.K8s), - Action: new(GetAllNodesK8sVersion), - Parallel: true, - } - - calculateMinK8sVersion := &task.LocalTask{ - Name: "CalculateMinK8sVersion", - Desc: "Calculate min Kubernetes version", - Action: new(precheck.CalculateMinK8sVersion), - } - - calculateMaxK8sVersion := &task.LocalTask{ - Name: "CalculateMaxK8sVersion", - Desc: "Calculate max Kubernetes version", - Action: new(CalculateMaxK8sVersion), - } - - checkDesiredK8sVersion := &task.LocalTask{ - Name: "CheckDesiredK8sVersion", - Desc: "Check desired Kubernetes version", - Action: new(precheck.CheckDesiredK8sVersion), - } - - checkUpgradeK8sVersion := &task.LocalTask{ - Name: "checkUpgradeK8sVersion", - Desc: "Check the Kubernetes version can correctly upgrade", - Action: new(CheckUpgradeK8sVersion), - } - - ksVersionCheck := &task.RemoteTask{ - Name: "KsVersionCheck", - Desc: "Check KubeSphere version", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(precheck.KsVersionCheck), - Parallel: true, - } - - getKubernetesNodesStatus := &task.RemoteTask{ - Name: "GetKubernetesNodesStatus", - Desc: "Get kubernetes nodes status", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(precheck.GetKubernetesNodesStatus), - Parallel: true, - } - - c.Tasks = []task.Interface{ - nodePreCheck, - getKubeConfig, - getAllNodesK8sVersion, - calculateMinK8sVersion, - calculateMaxK8sVersion, - checkDesiredK8sVersion, - checkUpgradeK8sVersion, - ksVersionCheck, - getKubernetesNodesStatus, - } -} - -type UpgradeKubeSpherePreCheckModule struct { - common.KubeModule -} - -func (c *UpgradeKubeSpherePreCheckModule) Init() { - c.Name = "UpgradeKubeSpherePreCheckModule" - c.Desc = "Do pre-check on for upgrade kubesphere phase" - - nodePreCheck := &task.RemoteTask{ - Name: "NodePreCheck", - Desc: "A pre-check on nodes", - Hosts: c.Runtime.GetAllHosts(), - Action: new(precheck.NodePreCheck), - Parallel: true, - } - - getKubeConfig := &task.RemoteTask{ - Name: "GetKubeConfig", - Desc: "Get KubeConfig file", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(precheck.GetKubeConfig), - Parallel: true, - } - - getMasterK8sVersion := &task.RemoteTask{ - Name: "GetMasterK8sVersion", - Desc: "get the master Kubernetes version", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(GetMasterK8sVersion), - } - - ksVersionCheck := &task.RemoteTask{ - Name: "KsVersionCheck", - Desc: "Check KubeSphere version", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(precheck.KsVersionCheck), - Parallel: true, - } - - c.Tasks = []task.Interface{ - nodePreCheck, - getKubeConfig, - getMasterK8sVersion, - ksVersionCheck, - } -} - -type UpgradeksPhaseDependencyCheckModule struct { - common.KubeModule -} - -func (c *UpgradeksPhaseDependencyCheckModule) Init() { - c.Name = "UpgradeksPhaseDependencyCheckModule" - c.Desc = "Check dependency matrix for KubeSphere and Kubernetes in ks phase" - - ksPhaseDependencyCheck := &task.RemoteTask{ - Name: "ksPhaseDependencyCheck", - Desc: "Check dependency matrix for KubeSphere and Kubernetes in ks phase", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(precheck.KubeSphereExist), - }, - Action: new(KsPhaseDependencyCheck), - } - - c.Tasks = []task.Interface{ - ksPhaseDependencyCheck, - } -} diff --git a/cmd/kk/pkg/phase/precheck/task.go b/cmd/kk/pkg/phase/precheck/task.go deleted file mode 100644 index 603956cb0..000000000 --- a/cmd/kk/pkg/phase/precheck/task.go +++ /dev/null @@ -1,188 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package precheck - -import ( - "fmt" - "strings" - - "github.com/pkg/errors" - versionutil "k8s.io/apimachinery/pkg/util/version" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere" -) - -type CheckUpgradeK8sVersion struct { - common.KubeAction -} - -func compareVersionsForUpgrade(currentVersion *versionutil.Version, currentMaxVersion *versionutil.Version, desiredVersion *versionutil.Version) error { - if desiredVersion.LessThan(currentMaxVersion) { - return errors.New("desired version to upgrade is less than the max version in cluster") - } - if desiredVersion.Major()-currentVersion.Major() != 0 || desiredVersion.Minor()-currentVersion.Minor() > 1 { - return errors.New("skipping MINOR versions when upgrading is unsupported") - } - return nil -} - -func (k *CheckUpgradeK8sVersion) Execute(_ connector.Runtime) error { - minK8sVersion, ok := k.PipelineCache.GetMustString(common.K8sVersion) - if !ok { - return errors.New("get current k8s version failed by pipeline cache") - } - minVersionObj, err := versionutil.ParseSemantic(minK8sVersion) - if err != nil { - return errors.Wrap(err, "parse min k8s version failed") - } - - maxK8sVersion, ok := k.PipelineCache.GetMustString(common.MaxK8sVersion) - if !ok { - return errors.New("get max k8s version failed by pipeline cache") - } - maxVersionObj, err := versionutil.ParseSemantic(maxK8sVersion) - if err != nil { - return errors.Wrap(err, "parse max k8s version failed") - } - - desiredVersion, ok := k.PipelineCache.GetMustString(common.DesiredK8sVersion) - if !ok { - return errors.New("get desired k8s version failed by pipeline cache") - } - desiredVersionObj, err := versionutil.ParseSemantic(desiredVersion) - if err != nil { - return errors.Wrap(err, "parse desired k8s version failed") - } - - if err := compareVersionsForUpgrade(minVersionObj, maxVersionObj, desiredVersionObj); err != nil { - return err - } - - k.PipelineCache.Set(common.PlanK8sVersion, desiredVersion) - return nil -} - -type CalculateMaxK8sVersion struct { - common.KubeAction -} - -func (g *CalculateMaxK8sVersion) Execute(runtime connector.Runtime) error { - versionList := make([]*versionutil.Version, 0, len(runtime.GetHostsByRole(common.K8s))) - for _, host := range runtime.GetHostsByRole(common.K8s) { - version, ok := host.GetCache().GetMustString(common.NodeK8sVersion) - if !ok { - return errors.Errorf("get node %s Kubernetes version failed by host cache", host.GetName()) - } - if versionObj, err := versionutil.ParseSemantic(version); err != nil { - return errors.Wrap(err, "parse node version failed") - } else { - versionList = append(versionList, versionObj) - } - } - - maxVersion := versionList[0] - for _, version := range versionList { - if maxVersion.LessThan(version) { - maxVersion = version - } - } - g.PipelineCache.Set(common.MaxK8sVersion, fmt.Sprintf("v%s", maxVersion)) - return nil -} - -type GetAllNodesK8sVersion struct { - common.KubeAction -} - -func (g *GetAllNodesK8sVersion) Execute(runtime connector.Runtime) error { - var nodeK8sVersion string - host := runtime.RemoteHost() - hostName := host.GetName() - - kubectlCmd := fmt.Sprintf("/usr/local/bin/kubectl get nodes %s", hostName) - kubectlVersionInfo, err := runtime.GetRunner().SudoCmd(kubectlCmd, false) - if err != nil { - errorMsg := fmt.Sprintf("get current version of %s failed", hostName) - return errors.Wrap(err, errorMsg) - } - - kubectlCmdGroup := strings.Split(kubectlVersionInfo, " ") - nodeK8sVersion = kubectlCmdGroup[len(kubectlCmdGroup)-1] - - if host.IsRole(common.Master) { - apiserverVersion, err := runtime.GetRunner().SudoCmd( - "cat /etc/kubernetes/manifests/kube-apiserver.yaml | grep 'image:' | rev | cut -d ':' -f1 | rev", - false) - if err != nil { - return errors.Wrap(err, "get current kube-apiserver version failed") - } - nodeK8sVersion = apiserverVersion - } - host.GetCache().Set(common.NodeK8sVersion, nodeK8sVersion) - return nil -} - -type GetMasterK8sVersion struct { - common.KubeAction -} - -func (g *GetMasterK8sVersion) Execute(runtime connector.Runtime) error { - apiserverVersion, err := runtime.GetRunner().SudoCmd( - "cat /etc/kubernetes/manifests/kube-apiserver.yaml | grep 'image:' | rev | cut -d ':' -f1 | rev", - false) - if err != nil { - return errors.Wrap(err, "get current kube-apiserver version failed") - } - g.PipelineCache.Set(common.K8sVersion, apiserverVersion) - return nil -} - -type KsPhaseDependencyCheck struct { - common.KubeAction -} - -func (d *KsPhaseDependencyCheck) Execute(_ connector.Runtime) error { - currentKsVersion, ok := d.PipelineCache.GetMustString(common.KubeSphereVersion) - if !ok { - return errors.New("get current KubeSphere version failed by pipeline cache") - } - currentK8sVersion, ok := d.PipelineCache.GetMustString(common.K8sVersion) - if !ok { - return errors.New("get current K8s version failed by pipeline cache") - } - desiredVersion := d.KubeConf.Cluster.KubeSphere.Version - - if d.KubeConf.Cluster.KubeSphere.Enabled { - KsInstaller, ok := kubesphere.VersionMap[desiredVersion] - if !ok { - return errors.New(fmt.Sprintf("Unsupported version: %s", desiredVersion)) - } - if currentKsVersion != desiredVersion { - if ok := KsInstaller.UpgradeSupport(currentKsVersion); !ok { - return errors.New(fmt.Sprintf("Unsupported upgrade plan: %s to %s", currentKsVersion, desiredVersion)) - } - } - if ok := KsInstaller.K8sSupport(currentK8sVersion); !ok { - return errors.New(fmt.Sprintf("KubeSphere %s does not support running on Kubernetes %s", - currentK8sVersion, d.KubeConf.Cluster.Kubernetes.Version)) - } - } - - return nil -} diff --git a/cmd/kk/pkg/pipelines/add_nodes.go b/cmd/kk/pkg/pipelines/add_nodes.go deleted file mode 100644 index 61dd067e0..000000000 --- a/cmd/kk/pkg/pipelines/add_nodes.go +++ /dev/null @@ -1,199 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pipelines - -import ( - "fmt" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/artifact" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/binaries" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/confirm" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/customscripts" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/registry" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/certs" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/etcd" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/filesystem" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/k3s" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/k8e" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/loadbalancer" -) - -func NewAddNodesPipeline(runtime *common.KubeRuntime) error { - noArtifact := runtime.Arg.Artifact == "" - - m := []module.Module{ - &precheck.GreetingsModule{}, - &customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: runtime.Cluster.System.PreInstall}, - &precheck.NodePreCheckModule{}, - &confirm.InstallConfirmModule{}, - &artifact.UnArchiveModule{Skip: noArtifact}, - &os.RepositoryModule{Skip: noArtifact || !runtime.Arg.InstallPackages}, - &binaries.NodeBinariesModule{}, - &os.ConfigureOSModule{Skip: runtime.Cluster.System.SkipConfigureOS}, - ®istry.RegistryCertsModule{Skip: len(runtime.GetHostsByRole(common.Registry)) == 0}, - //for one master to multi master kube-vip - &loadbalancer.KubevipModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabledVip()}, - &kubernetes.RestartKubeletModule{}, - &kubernetes.StatusModule{}, - &container.InstallContainerModule{}, - &images.PullModule{Skip: runtime.Arg.SkipPullImages}, - &etcd.PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.CertsModule{}, - &etcd.InstallETCDBinaryModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.ConfigureModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.BackupModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &kubernetes.InstallKubeBinariesModule{}, - &kubernetes.JoinNodesModule{}, - &loadbalancer.HaproxyModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabled()}, - &kubernetes.ConfigureKubernetesModule{}, - &filesystem.ChownModule{}, - &certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()}, - &customscripts.CustomScriptsModule{Phase: "PostInstall", Scripts: runtime.Cluster.System.PostInstall}, - } - - p := pipeline.Pipeline{ - Name: "AddNodesPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - - return nil -} - -func NewK3sAddNodesPipeline(runtime *common.KubeRuntime) error { - noArtifact := runtime.Arg.Artifact == "" - - m := []module.Module{ - &precheck.GreetingsModule{}, - &artifact.UnArchiveModule{Skip: noArtifact}, - &os.RepositoryModule{Skip: noArtifact || !runtime.Arg.InstallPackages}, - &binaries.K3sNodeBinariesModule{}, - &os.ConfigureOSModule{Skip: runtime.Cluster.System.SkipConfigureOS}, - &customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: runtime.Cluster.System.PreInstall}, - &k3s.StatusModule{}, - &etcd.PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.CertsModule{}, - &etcd.InstallETCDBinaryModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.ConfigureModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.BackupModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &k3s.InstallKubeBinariesModule{}, - &k3s.JoinNodesModule{}, - &loadbalancer.K3sHaproxyModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabled()}, - &kubernetes.ConfigureKubernetesModule{}, - &filesystem.ChownModule{}, - &certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()}, - &customscripts.CustomScriptsModule{Phase: "PostInstall", Scripts: runtime.Cluster.System.PostInstall}, - } - - p := pipeline.Pipeline{ - Name: "AddNodesPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - - return nil -} - -func NewK8eAddNodesPipeline(runtime *common.KubeRuntime) error { - noArtifact := runtime.Arg.Artifact == "" - - m := []module.Module{ - &precheck.GreetingsModule{}, - &artifact.UnArchiveModule{Skip: noArtifact}, - &os.RepositoryModule{Skip: noArtifact || !runtime.Arg.InstallPackages}, - &binaries.K8eNodeBinariesModule{}, - &os.ConfigureOSModule{Skip: runtime.Cluster.System.SkipConfigureOS}, - - &k8e.StatusModule{}, - &etcd.PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.CertsModule{}, - &etcd.InstallETCDBinaryModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.ConfigureModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.BackupModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &k8e.InstallKubeBinariesModule{}, - &k8e.JoinNodesModule{}, - &loadbalancer.K3sHaproxyModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabled()}, - &kubernetes.ConfigureKubernetesModule{}, - &filesystem.ChownModule{}, - &certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()}, - } - - p := pipeline.Pipeline{ - Name: "AddNodesPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - - return nil -} - -func AddNodes(args common.Argument, downloadCmd string) error { - args.DownloadCommand = func(path, url string) string { - // this is an extension point for downloading tools, for example users can set the timeout, proxy or retry under - // some poor network environment. Or users even can choose another cli, it might be wget. - // perhaps we should have a build-in download function instead of totally rely on the external one - return fmt.Sprintf(downloadCmd, path, url) - } - - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - switch runtime.Cluster.Kubernetes.Type { - case common.K3s: - if err := NewK3sAddNodesPipeline(runtime); err != nil { - return err - } - case common.K8e: - if err := NewK8eAddNodesPipeline(runtime); err != nil { - return err - } - case common.Kubernetes: - fallthrough - default: - if err := NewAddNodesPipeline(runtime); err != nil { - return err - } - } - return nil -} diff --git a/cmd/kk/pkg/pipelines/artifact_export.go b/cmd/kk/pkg/pipelines/artifact_export.go deleted file mode 100644 index 6e730deb5..000000000 --- a/cmd/kk/pkg/pipelines/artifact_export.go +++ /dev/null @@ -1,148 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pipelines - -import ( - "fmt" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/artifact" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/binaries" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/confirm" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/filesystem" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" -) - -func NewArtifactExportPipeline(runtime *common.ArtifactRuntime) error { - m := []module.Module{ - &confirm.CheckFileExistModule{FileName: runtime.Arg.Output}, - &images.CopyImagesToLocalModule{}, - &binaries.ArtifactBinariesModule{}, - &artifact.RepositoryModule{}, - &artifact.ArchiveModule{}, - &filesystem.ChownOutputModule{}, - &filesystem.ChownWorkDirModule{}, - } - - p := pipeline.Pipeline{ - Name: "ArtifactExportPipeline", - Modules: m, - Runtime: runtime, - ModulePostHooks: nil, - } - if err := p.Start(); err != nil { - return err - } - - return nil -} - -func NewK3sArtifactExportPipeline(runtime *common.ArtifactRuntime) error { - m := []module.Module{ - &confirm.CheckFileExistModule{FileName: runtime.Arg.Output}, - &images.CopyImagesToLocalModule{}, - &binaries.K3sArtifactBinariesModule{}, - &artifact.RepositoryModule{}, - &artifact.ArchiveModule{}, - &filesystem.ChownOutputModule{}, - &filesystem.ChownWorkDirModule{}, - } - - p := pipeline.Pipeline{ - Name: "K3sArtifactExportPipeline", - Modules: m, - Runtime: runtime, - ModulePostHooks: nil, - } - if err := p.Start(); err != nil { - return err - } - - return nil -} - -func NewK8eArtifactExportPipeline(runtime *common.ArtifactRuntime) error { - m := []module.Module{ - &confirm.CheckFileExistModule{FileName: runtime.Arg.Output}, - &images.CopyImagesToLocalModule{}, - &binaries.K8eArtifactBinariesModule{}, - &artifact.RepositoryModule{}, - &artifact.ArchiveModule{}, - &filesystem.ChownOutputModule{}, - &filesystem.ChownWorkDirModule{}, - } - - p := pipeline.Pipeline{ - Name: "K8eArtifactBinariesModule", - Modules: m, - Runtime: runtime, - ModulePostHooks: nil, - } - if err := p.Start(); err != nil { - return err - } - - return nil -} - -func ArtifactExport(args common.ArtifactArgument, downloadCmd string) error { - args.DownloadCommand = func(path, url string) string { - // this is an extension point for downloading tools, for example users can set the timeout, proxy or retry under - // some poor network environment. Or users even can choose another cli, it might be wget. - // perhaps we should have a build-in download function instead of totally rely on the external one - return fmt.Sprintf(downloadCmd, path, url) - } - - runtime, err := common.NewArtifactRuntime(args) - if err != nil { - return err - } - - if len(runtime.Spec.KubernetesDistributions) == 0 { - return errors.New("the length of kubernetes distributions can't be 0") - } - - pre := runtime.Spec.KubernetesDistributions[0].Type - for _, t := range runtime.Spec.KubernetesDistributions { - if t.Type != pre { - return errors.New("all the types of kubernetes distributions can't be different") - } - } - - switch runtime.Spec.KubernetesDistributions[0].Type { - case common.K3s: - if err := NewK3sArtifactExportPipeline(runtime); err != nil { - return err - } - case common.K8e: - if err := NewK8eArtifactExportPipeline(runtime); err != nil { - return err - } - case common.Kubernetes: - fallthrough - default: - if err := NewArtifactExportPipeline(runtime); err != nil { - return err - } - } - - return nil -} diff --git a/cmd/kk/pkg/pipelines/check_certs.go b/cmd/kk/pkg/pipelines/check_certs.go deleted file mode 100644 index 261a26e4a..000000000 --- a/cmd/kk/pkg/pipelines/check_certs.go +++ /dev/null @@ -1,62 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pipelines - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/certs" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" -) - -func CheckCertsPipeline(runtime *common.KubeRuntime) error { - m := []module.Module{ - &precheck.GreetingsModule{}, - &certs.CheckCertsModule{}, - &certs.PrintClusterCertsModule{}, - } - - p := pipeline.Pipeline{ - Name: "CheckCertsPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func CheckCerts(args common.Argument) error { - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - if err := CheckCertsPipeline(runtime); err != nil { - return err - } - return nil -} diff --git a/cmd/kk/pkg/pipelines/create_cluster.go b/cmd/kk/pkg/pipelines/create_cluster.go deleted file mode 100644 index 99fc3e51b..000000000 --- a/cmd/kk/pkg/pipelines/create_cluster.go +++ /dev/null @@ -1,318 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pipelines - -import ( - "fmt" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/addons" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/artifact" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/binaries" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/confirm" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/customscripts" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/certs" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/etcd" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/filesystem" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/k3s" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/k8e" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubesphere" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/loadbalancer" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/dns" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/network" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/storage" -) - -func NewCreateClusterPipeline(runtime *common.KubeRuntime) error { - noArtifact := runtime.Arg.Artifact == "" - skipPushImages := runtime.Arg.SkipPushImages || noArtifact || (!noArtifact && runtime.Cluster.Registry.PrivateRegistry == "") - skipLocalStorage := true - if runtime.Arg.DeployLocalStorage != nil { - skipLocalStorage = !*runtime.Arg.DeployLocalStorage - } else if runtime.Cluster.KubeSphere.Enabled { - skipLocalStorage = false - } - - m := []module.Module{ - &precheck.GreetingsModule{}, - &customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: runtime.Cluster.System.PreInstall}, - &precheck.NodePreCheckModule{}, - &confirm.InstallConfirmModule{}, - &artifact.UnArchiveModule{Skip: noArtifact}, - &os.RepositoryModule{Skip: noArtifact || !runtime.Arg.InstallPackages}, - &binaries.NodeBinariesModule{}, - &os.ConfigureOSModule{Skip: runtime.Cluster.System.SkipConfigureOS}, - &kubernetes.StatusModule{}, - &container.InstallContainerModule{}, - &images.CopyImagesToRegistryModule{Skip: skipPushImages}, - &images.PullModule{Skip: runtime.Arg.SkipPullImages}, - &etcd.PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.CertsModule{}, - &etcd.InstallETCDBinaryModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.ConfigureModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.BackupModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &kubernetes.InstallKubeBinariesModule{}, - // init kubeVip on first master - &loadbalancer.KubevipModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabledVip()}, - &kubernetes.InitKubernetesModule{}, - &dns.ClusterDNSModule{}, - &kubernetes.StatusModule{}, - &kubernetes.JoinNodesModule{}, - // deploy kubeVip on other masters - &loadbalancer.KubevipModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabledVip()}, - &loadbalancer.HaproxyModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabled()}, - &network.DeployNetworkPluginModule{}, - &kubernetes.ConfigureKubernetesModule{}, - &filesystem.ChownModule{}, - &certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()}, - &kubernetes.SecurityEnhancementModule{Skip: !runtime.Arg.SecurityEnhancement}, - &kubernetes.SaveKubeConfigModule{}, - &plugins.DeployPluginsModule{}, - &addons.AddonsModule{}, - &storage.DeployLocalVolumeModule{Skip: skipLocalStorage}, - &kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &kubesphere.CheckResultModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &customscripts.CustomScriptsModule{Phase: "PostInstall", Scripts: runtime.Cluster.System.PostInstall}, - } - - p := pipeline.Pipeline{ - Name: "CreateClusterPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - - if runtime.Cluster.KubeSphere.Enabled { - - fmt.Print(`Installation is complete. - -Please check the result using the command: - - kubectl logs -n kubesphere-system $(kubectl get pod -n kubesphere-system -l 'app in (ks-install, ks-installer)' -o jsonpath='{.items[0].metadata.name}') -f - -`) - } else { - fmt.Print(`Installation is complete. - -Please check the result using the command: - - kubectl get pod -A - -`) - - } - - return nil -} - -func NewK3sCreateClusterPipeline(runtime *common.KubeRuntime) error { - noArtifact := runtime.Arg.Artifact == "" - skipPushImages := runtime.Arg.SkipPushImages || noArtifact || (!noArtifact && runtime.Cluster.Registry.PrivateRegistry == "") - skipLocalStorage := true - if runtime.Arg.DeployLocalStorage != nil { - skipLocalStorage = !*runtime.Arg.DeployLocalStorage - } else if runtime.Cluster.KubeSphere.Enabled { - skipLocalStorage = false - } - - m := []module.Module{ - &precheck.GreetingsModule{}, - &artifact.UnArchiveModule{Skip: noArtifact}, - &os.RepositoryModule{Skip: noArtifact || !runtime.Arg.InstallPackages}, - &binaries.K3sNodeBinariesModule{}, - &os.ConfigureOSModule{Skip: runtime.Cluster.System.SkipConfigureOS}, - &customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: runtime.Cluster.System.PreInstall}, - &k3s.StatusModule{}, - &etcd.PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.CertsModule{}, - &etcd.InstallETCDBinaryModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.ConfigureModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.BackupModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &loadbalancer.K3sKubevipModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabledVip()}, - &k3s.InstallKubeBinariesModule{}, - &k3s.InitClusterModule{}, - &k3s.StatusModule{}, - &k3s.JoinNodesModule{}, - &images.CopyImagesToRegistryModule{Skip: skipPushImages}, - &loadbalancer.K3sHaproxyModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabled()}, - &network.DeployNetworkPluginModule{}, - &kubernetes.ConfigureKubernetesModule{}, - &filesystem.ChownModule{}, - &certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()}, - &k3s.SaveKubeConfigModule{}, - &addons.AddonsModule{}, - &storage.DeployLocalVolumeModule{Skip: skipLocalStorage}, - &kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &kubesphere.CheckResultModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &customscripts.CustomScriptsModule{Phase: "PostInstall", Scripts: runtime.Cluster.System.PostInstall}, - } - - p := pipeline.Pipeline{ - Name: "K3sCreateClusterPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - - if runtime.Cluster.KubeSphere.Enabled { - - fmt.Print(`Installation is complete. - -Please check the result using the command: - - kubectl logs -n kubesphere-system $(kubectl get pod -n kubesphere-system -l 'app in (ks-install, ks-installer)' -o jsonpath='{.items[0].metadata.name}') -f - -`) - } else { - fmt.Print(`Installation is complete. - -Please check the result using the command: - - kubectl get pod -A - -`) - - } - - return nil -} - -func NewK8eCreateClusterPipeline(runtime *common.KubeRuntime) error { - noArtifact := runtime.Arg.Artifact == "" - skipPushImages := runtime.Arg.SkipPushImages || noArtifact || (!noArtifact && runtime.Cluster.Registry.PrivateRegistry == "") - skipLocalStorage := true - if runtime.Arg.DeployLocalStorage != nil { - skipLocalStorage = !*runtime.Arg.DeployLocalStorage - } else if runtime.Cluster.KubeSphere.Enabled { - skipLocalStorage = false - } - - m := []module.Module{ - &precheck.GreetingsModule{}, - &artifact.UnArchiveModule{Skip: noArtifact}, - &os.RepositoryModule{Skip: noArtifact || !runtime.Arg.InstallPackages}, - &binaries.K8eNodeBinariesModule{}, - &os.ConfigureOSModule{Skip: runtime.Cluster.System.SkipConfigureOS}, - &customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: runtime.Cluster.System.PreInstall}, - &k8e.StatusModule{}, - &etcd.PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.CertsModule{}, - &etcd.InstallETCDBinaryModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.ConfigureModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.BackupModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &loadbalancer.K3sKubevipModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabledVip()}, - &k8e.InstallKubeBinariesModule{}, - &k8e.InitClusterModule{}, - &k8e.StatusModule{}, - &k8e.JoinNodesModule{}, - &images.CopyImagesToRegistryModule{Skip: skipPushImages}, - &loadbalancer.K3sHaproxyModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabled()}, - &network.DeployNetworkPluginModule{}, - &kubernetes.ConfigureKubernetesModule{}, - &filesystem.ChownModule{}, - &certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()}, - &k8e.SaveKubeConfigModule{}, - &addons.AddonsModule{}, - &storage.DeployLocalVolumeModule{Skip: skipLocalStorage}, - &kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &kubesphere.CheckResultModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &customscripts.CustomScriptsModule{Phase: "PostInstall", Scripts: runtime.Cluster.System.PostInstall}, - } - - p := pipeline.Pipeline{ - Name: "K8eCreateClusterPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - - if runtime.Cluster.KubeSphere.Enabled { - - fmt.Print(`Installation is complete. - -Please check the result using the command: - - kubectl logs -n kubesphere-system $(kubectl get pod -n kubesphere-system -l 'app in (ks-install, ks-installer)' -o jsonpath='{.items[0].metadata.name}') -f - -`) - } else { - fmt.Print(`Installation is complete. - -Please check the result using the command: - - kubectl get pod -A - -`) - - } - - return nil -} - -func CreateCluster(args common.Argument, downloadCmd string) error { - args.DownloadCommand = func(path, url string) string { - // this is an extension point for downloading tools, for example users can set the timeout, proxy or retry under - // some poor network environment. Or users even can choose another cli, it might be wget. - // perhaps we should have a build-in download function instead of totally rely on the external one - return fmt.Sprintf(downloadCmd, path, url) - } - - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - switch runtime.Cluster.Kubernetes.Type { - case common.K3s: - if err := NewK3sCreateClusterPipeline(runtime); err != nil { - return err - } - case common.K8e: - if err := NewK8eCreateClusterPipeline(runtime); err != nil { - return err - } - case common.Kubernetes: - fallthrough - default: - if err := NewCreateClusterPipeline(runtime); err != nil { - return err - } - } - return nil -} diff --git a/cmd/kk/pkg/pipelines/delete_cluster.go b/cmd/kk/pkg/pipelines/delete_cluster.go deleted file mode 100644 index e93def1f0..000000000 --- a/cmd/kk/pkg/pipelines/delete_cluster.go +++ /dev/null @@ -1,127 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pipelines - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/confirm" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/certs" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/k3s" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/k8e" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/loadbalancer" -) - -func NewDeleteClusterPipeline(runtime *common.KubeRuntime) error { - m := []module.Module{ - &precheck.GreetingsModule{}, - &confirm.DeleteClusterConfirmModule{Skip: runtime.Arg.SkipConfirmCheck}, - &kubernetes.ResetClusterModule{}, - &container.UninstallContainerModule{Skip: !runtime.Arg.DeleteCRI}, - &os.ClearOSEnvironmentModule{}, - &certs.UninstallAutoRenewCertsModule{}, - &loadbalancer.DeleteVIPModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabledVip()}, - } - - p := pipeline.Pipeline{ - Name: "DeleteClusterPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func NewK3sDeleteClusterPipeline(runtime *common.KubeRuntime) error { - m := []module.Module{ - &precheck.GreetingsModule{}, - &confirm.DeleteClusterConfirmModule{}, - &k3s.DeleteClusterModule{}, - &os.ClearOSEnvironmentModule{}, - &certs.UninstallAutoRenewCertsModule{}, - &loadbalancer.DeleteVIPModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabledVip()}, - } - - p := pipeline.Pipeline{ - Name: "K3sDeleteClusterPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func NewK8eDeleteClusterPipeline(runtime *common.KubeRuntime) error { - m := []module.Module{ - &precheck.GreetingsModule{}, - &confirm.DeleteClusterConfirmModule{}, - &k8e.DeleteClusterModule{}, - &os.ClearOSEnvironmentModule{}, - &certs.UninstallAutoRenewCertsModule{}, - } - - p := pipeline.Pipeline{ - Name: "K8eDeleteClusterPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func DeleteCluster(args common.Argument) error { - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - switch runtime.Cluster.Kubernetes.Type { - case common.K3s: - if err := NewK3sDeleteClusterPipeline(runtime); err != nil { - return err - } - case common.K8e: - if err := NewK8eDeleteClusterPipeline(runtime); err != nil { - return err - } - case common.Kubernetes: - fallthrough - default: - if err := NewDeleteClusterPipeline(runtime); err != nil { - return err - } - } - return nil -} diff --git a/cmd/kk/pkg/pipelines/delete_node.go b/cmd/kk/pkg/pipelines/delete_node.go deleted file mode 100644 index e55ae7a9f..000000000 --- a/cmd/kk/pkg/pipelines/delete_node.go +++ /dev/null @@ -1,68 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pipelines - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/confirm" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/loadbalancer" -) - -func DeleteNodePipeline(runtime *common.KubeRuntime) error { - m := []module.Module{ - &precheck.GreetingsModule{}, - &confirm.DeleteNodeConfirmModule{Skip: runtime.Arg.SkipConfirmCheck}, - &kubernetes.CompareConfigAndClusterInfoModule{}, - &kubernetes.DeleteKubeNodeModule{}, - &os.ClearNodeOSModule{}, - &loadbalancer.DeleteVIPModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabledVip()}, - } - - p := pipeline.Pipeline{ - Name: "DeleteNodePipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func DeleteNode(args common.Argument) error { - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - if err := DeleteNodePipeline(runtime); err != nil { - return err - } - return nil -} diff --git a/cmd/kk/pkg/pipelines/init_dependencies.go b/cmd/kk/pkg/pipelines/init_dependencies.go deleted file mode 100644 index 0f8c6871c..000000000 --- a/cmd/kk/pkg/pipelines/init_dependencies.go +++ /dev/null @@ -1,70 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pipelines - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/artifact" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/customscripts" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/filesystem" -) - -func NewInitDependenciesPipeline(runtime *common.KubeRuntime) error { - noArtifact := runtime.Arg.Artifact == "" - - m := []module.Module{ - &precheck.GreetingsModule{}, - &artifact.UnArchiveModule{Skip: noArtifact}, - &os.RepositoryModule{Skip: noArtifact}, - &os.RepositoryOnlineModule{Skip: !noArtifact}, - &filesystem.ChownWorkDirModule{}, - &customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: runtime.Cluster.System.PreInstall}, - } - - p := pipeline.Pipeline{ - Name: "InitDependenciesPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func InitDependencies(args common.Argument) error { - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - if err := NewInitDependenciesPipeline(runtime); err != nil { - return err - } - return nil -} diff --git a/cmd/kk/pkg/pipelines/init_registry.go b/cmd/kk/pkg/pipelines/init_registry.go deleted file mode 100644 index 38c26c766..000000000 --- a/cmd/kk/pkg/pipelines/init_registry.go +++ /dev/null @@ -1,86 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pipelines - -import ( - "fmt" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/artifact" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/binaries" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/registry" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/filesystem" - "github.com/pkg/errors" -) - -func NewInitRegistryPipeline(runtime *common.KubeRuntime) error { - noArtifact := runtime.Arg.Artifact == "" - - m := []module.Module{ - &precheck.GreetingsModule{}, - &artifact.UnArchiveModule{Skip: noArtifact}, - &binaries.RegistryPackageModule{}, - &os.ConfigureOSModule{Skip: runtime.Cluster.System.SkipConfigureOS}, - ®istry.RegistryCertsModule{}, - ®istry.InstallRegistryModule{}, - &filesystem.ChownWorkDirModule{}, - } - - p := pipeline.Pipeline{ - Name: "InitRegistryPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func InitRegistry(args common.Argument, downloadCmd string) error { - args.DownloadCommand = func(path, url string) string { - // this is an extension point for downloading tools, for example users can set the timeout, proxy or retry under - // some poor network environment. Or users even can choose another cli, it might be wget. - // perhaps we should have a build-in download function instead of totally rely on the external one - return fmt.Sprintf(downloadCmd, path, url) - } - - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - if len(runtime.GetHostsByRole("registry")) <= 0 { - logger.Log.Fatal(errors.New("The number of registry must be greater then 0.")) - } - - if err := NewInitRegistryPipeline(runtime); err != nil { - return err - } - return nil -} diff --git a/cmd/kk/pkg/pipelines/migrate_container.go b/cmd/kk/pkg/pipelines/migrate_container.go deleted file mode 100644 index 0378e5ad0..000000000 --- a/cmd/kk/pkg/pipelines/migrate_container.go +++ /dev/null @@ -1,68 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pipelines - -import ( - "fmt" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/binaries" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/confirm" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" -) - -func MigrateCriPipeline(runtime *common.KubeRuntime) error { - fmt.Println("MigrateContainerdPipeline called") - m := []module.Module{ - &precheck.GreetingsModule{}, - &confirm.MigrateCriConfirmModule{}, - &binaries.CriBinariesModule{}, - &container.CriMigrateModule{}, - } - p := pipeline.Pipeline{ - Name: "MigrateContainerdPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func MigrateCri(args common.Argument, downloadCmd string) error { - args.DownloadCommand = func(path, url string) string { - return fmt.Sprintf(downloadCmd, path, url) - } - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - if err := MigrateCriPipeline(runtime); err != nil { - return err - } - return nil -} diff --git a/cmd/kk/pkg/pipelines/renew_certs.go b/cmd/kk/pkg/pipelines/renew_certs.go deleted file mode 100644 index f2753ebad..000000000 --- a/cmd/kk/pkg/pipelines/renew_certs.go +++ /dev/null @@ -1,63 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pipelines - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/certs" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" -) - -func RenewCertsPipeline(runtime *common.KubeRuntime) error { - m := []module.Module{ - &precheck.GreetingsModule{}, - &certs.RenewCertsModule{}, - &certs.CheckCertsModule{}, - &certs.PrintClusterCertsModule{}, - } - - p := pipeline.Pipeline{ - Name: "RenewCertsPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func RenewCerts(args common.Argument) error { - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - if err := RenewCertsPipeline(runtime); err != nil { - return err - } - return nil -} diff --git a/cmd/kk/pkg/pipelines/upgrade_cluster.go b/cmd/kk/pkg/pipelines/upgrade_cluster.go deleted file mode 100644 index 8aafd4f0f..000000000 --- a/cmd/kk/pkg/pipelines/upgrade_cluster.go +++ /dev/null @@ -1,100 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pipelines - -import ( - "fmt" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/artifact" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/confirm" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/certs" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/filesystem" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubesphere" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/loadbalancer" -) - -func NewUpgradeClusterPipeline(runtime *common.KubeRuntime) error { - noArtifact := runtime.Arg.Artifact == "" - - m := []module.Module{ - &precheck.GreetingsModule{}, - &precheck.NodePreCheckModule{}, - &precheck.ClusterPreCheckModule{}, - &confirm.UpgradeConfirmModule{Skip: runtime.Arg.SkipConfirmCheck}, - &artifact.UnArchiveModule{Skip: noArtifact}, - &kubernetes.SetUpgradePlanModule{Step: kubernetes.ToV121}, - &kubernetes.ProgressiveUpgradeModule{Step: kubernetes.ToV121}, - &loadbalancer.HaproxyModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabled()}, - &kubesphere.CleanClusterConfigurationModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &kubesphere.ConvertModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &kubesphere.CheckResultModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &kubernetes.SetUpgradePlanModule{Step: kubernetes.ToV122}, - &kubernetes.ProgressiveUpgradeModule{Step: kubernetes.ToV122}, - &filesystem.ChownModule{}, - &certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()}, - } - - p := pipeline.Pipeline{ - Name: "UpgradeClusterPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func UpgradeCluster(args common.Argument, downloadCmd string) error { - args.DownloadCommand = func(path, url string) string { - // this is an extension point for downloading tools, for example users can set the timeout, proxy or retry under - // some poor network environment. Or users even can choose another cli, it might be wget. - // perhaps we should have a build-in download function instead of totally rely on the external one - return fmt.Sprintf(downloadCmd, path, url) - } - - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewUpgradeClusterPipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/plugins/dns/module.go b/cmd/kk/pkg/plugins/dns/module.go deleted file mode 100644 index befb2baa6..000000000 --- a/cmd/kk/pkg/plugins/dns/module.go +++ /dev/null @@ -1,162 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package dns - -import ( - "path/filepath" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/dns/templates" -) - -type ClusterDNSModule struct { - common.KubeModule -} - -func (c *ClusterDNSModule) Init() { - c.Name = "ClusterDNSModule" - c.Desc = "Deploy cluster dns" - - generateCorednsConfigMap := &task.RemoteTask{ - Name: "GenerateCorednsConfigMap", - Desc: "Generate coredns configmap", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - }, - Action: &action.Template{ - Template: templates.CorednsConfigMap, - Dst: filepath.Join(common.KubeConfigDir, templates.CorednsConfigMap.Name()), - Data: util.Data{ - "DNSEtcHosts": c.KubeConf.Cluster.DNS.DNSEtcHosts, - "ExternalZones": c.KubeConf.Cluster.DNS.CoreDNS.ExternalZones, - "AdditionalConfigs": c.KubeConf.Cluster.DNS.CoreDNS.AdditionalConfigs, - "RewriteBlock": c.KubeConf.Cluster.DNS.CoreDNS.RewriteBlock, - "ClusterDomain": c.KubeConf.Cluster.Kubernetes.DNSDomain, - "UpstreamDNSServers": c.KubeConf.Cluster.DNS.CoreDNS.UpstreamDNSServers, - }, - }, - Parallel: true, - } - - applyCorednsConfigMap := &task.RemoteTask{ - Name: "ApplyCorednsConfigMap", - Desc: "Apply coredns configmap", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - }, - Action: new(ApplyCorednsConfigMap), - Parallel: true, - Retry: 5, - } - - generateCoreDNS := &task.RemoteTask{ - Name: "GenerateCoreDNS", - Desc: "Generate coredns manifests", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - }, - Action: new(GenerateCorednsmanifests), - Parallel: true, - } - - deployCoredns := &task.RemoteTask{ - Name: "DeployCoreDNS", - Desc: "Deploy coredns", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - }, - Action: new(DeployCoreDNS), - Parallel: true, - } - - generateNodeLocalDNSConfigMap := &task.RemoteTask{ - Name: "GenerateNodeLocalDNSConfigMap", - Desc: "Generate nodelocaldns configmap", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(EnableNodeLocalDNS), - }, - Action: new(GenerateNodeLocalDNSConfigMap), - Parallel: true, - } - - applyNodeLocalDNSConfigMap := &task.RemoteTask{ - Name: "ApplyNodeLocalDNSConfigMap", - Desc: "Apply nodelocaldns configmap", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(EnableNodeLocalDNS), - }, - Action: new(ApplyNodeLocalDNSConfigMap), - Parallel: true, - Retry: 5, - } - - generateNodeLocalDNS := &task.RemoteTask{ - Name: "GenerateNodeLocalDNS", - Desc: "Generate nodelocaldns", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(EnableNodeLocalDNS), - }, - Action: &action.Template{ - Template: templates.NodeLocalDNSService, - Dst: filepath.Join(common.KubeConfigDir, templates.NodeLocalDNSService.Name()), - Data: util.Data{ - "NodelocaldnsImage": images.GetImage(c.Runtime, c.KubeConf, "k8s-dns-node-cache").ImageName(), - "DNSEtcHosts": c.KubeConf.Cluster.DNS.DNSEtcHosts, - }, - }, - Parallel: true, - } - - applyNodeLocalDNS := &task.RemoteTask{ - Name: "DeployNodeLocalDNS", - Desc: "Deploy nodelocaldns", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(EnableNodeLocalDNS), - }, - Action: new(DeployNodeLocalDNS), - Parallel: true, - Retry: 5, - } - - c.Tasks = []task.Interface{ - generateCorednsConfigMap, - applyCorednsConfigMap, - generateCoreDNS, - deployCoredns, - generateNodeLocalDNSConfigMap, - applyNodeLocalDNSConfigMap, - generateNodeLocalDNS, - applyNodeLocalDNS, - } -} diff --git a/cmd/kk/pkg/plugins/dns/prepares.go b/cmd/kk/pkg/plugins/dns/prepares.go deleted file mode 100644 index 1b46cf555..000000000 --- a/cmd/kk/pkg/plugins/dns/prepares.go +++ /dev/null @@ -1,65 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package dns - -import ( - "strings" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type CoreDNSExist struct { - common.KubePrepare - Not bool -} - -func (c *CoreDNSExist) PreCheck(runtime connector.Runtime) (bool, error) { - _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl get svc -n kube-system coredns", false) - if err != nil { - if strings.Contains(err.Error(), "NotFound") { - return c.Not, nil - } - return false, err - } - return !c.Not, nil -} - -type EnableNodeLocalDNS struct { - common.KubePrepare -} - -func (e *EnableNodeLocalDNS) PreCheck(runtime connector.Runtime) (bool, error) { - if e.KubeConf.Cluster.Kubernetes.EnableNodelocaldns() { - return true, nil - } - return false, nil -} - -type NodeLocalDNSConfigMapNotExist struct { - common.KubePrepare -} - -func (n *NodeLocalDNSConfigMapNotExist) PreCheck(runtime connector.Runtime) (bool, error) { - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl get cm -n kube-system nodelocaldns", false); err != nil { - if strings.Contains(err.Error(), "NotFound") { - return true, nil - } - return false, err - } - return false, nil -} diff --git a/cmd/kk/pkg/plugins/dns/tasks.go b/cmd/kk/pkg/plugins/dns/tasks.go deleted file mode 100644 index 8a336d347..000000000 --- a/cmd/kk/pkg/plugins/dns/tasks.go +++ /dev/null @@ -1,133 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package dns - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/pkg/errors" - "path/filepath" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/dns/templates" -) - -type GenerateCorednsmanifests struct { - common.KubeAction -} - -func (g *GenerateCorednsmanifests) Execute(runtime connector.Runtime) error { - templateAction := action.Template{ - Template: templates.Coredns, - Dst: filepath.Join(common.KubeConfigDir, templates.Coredns.Name()), - Data: util.Data{ - "ClusterIP": g.KubeConf.Cluster.CorednsClusterIP(), - "CorednsImage": images.GetImage(runtime, g.KubeConf, "coredns").ImageName(), - "DNSEtcHosts": g.KubeConf.Cluster.DNS.DNSEtcHosts, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type DeployCoreDNS struct { - common.KubeAction -} - -func (o *DeployCoreDNS) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl delete svc -n kube-system --field-selector metadata.name=kube-dns", true); err != nil { - return errors.Wrap(errors.WithStack(err), "remove old coredns svc") - } - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/coredns.yaml", true); err != nil { - return errors.Wrap(errors.WithStack(err), "update coredns failed") - } - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl -n kube-system rollout restart deploy coredns", true); err != nil { - return errors.Wrap(errors.WithStack(err), "restart coredns failed") - } - return nil -} - -type ApplyCorednsConfigMap struct { - common.KubeAction -} - -func (o *ApplyCorednsConfigMap) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/coredns-configmap.yaml", true); err != nil { - return errors.Wrap(errors.WithStack(err), "create coredns configmap failed") - } - return nil -} - -type DeployNodeLocalDNS struct { - common.KubeAction -} - -func (d *DeployNodeLocalDNS) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/nodelocaldns.yaml", true); err != nil { - return errors.Wrap(errors.WithStack(err), "deploy nodelocaldns failed") - } - return nil -} - -type GenerateNodeLocalDNSConfigMap struct { - common.KubeAction -} - -func (g *GenerateNodeLocalDNSConfigMap) Execute(runtime connector.Runtime) error { - clusterIP, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl get svc -n kube-system coredns -o jsonpath='{.spec.clusterIP}'", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get clusterIP failed") - } - - if len(clusterIP) == 0 { - clusterIP = g.KubeConf.Cluster.CorednsClusterIP() - } - - templateAction := action.Template{ - Template: templates.NodeLocalDNSConfigMap, - Dst: filepath.Join(common.KubeConfigDir, templates.NodeLocalDNSConfigMap.Name()), - Data: util.Data{ - "ForwardTarget": clusterIP, - "DNSDomain": g.KubeConf.Cluster.Kubernetes.DNSDomain, - "ExternalZones": g.KubeConf.Cluster.DNS.NodeLocalDNS.ExternalZones, - "DNSEtcHosts": g.KubeConf.Cluster.DNS.DNSEtcHosts, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type ApplyNodeLocalDNSConfigMap struct { - common.KubeAction -} - -func (a *ApplyNodeLocalDNSConfigMap) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/nodelocaldns-configmap.yaml", true); err != nil { - return errors.Wrap(errors.WithStack(err), "apply nodelocaldns configmap failed") - } - return nil -} diff --git a/cmd/kk/pkg/plugins/dns/templates/coredns.go b/cmd/kk/pkg/plugins/dns/templates/coredns.go deleted file mode 100644 index 4443c153f..000000000 --- a/cmd/kk/pkg/plugins/dns/templates/coredns.go +++ /dev/null @@ -1,211 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var ( - Coredns = template.Must(template.New("coredns.yaml").Parse( - dedent.Dedent(`--- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - kubernetes.io/bootstrapping: rbac-defaults - addonmanager.kubernetes.io/mode: Reconcile - name: system:coredns -rules: -- apiGroups: - - "" - resources: - - endpoints - - services - - pods - - namespaces - verbs: - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - ---- -apiVersion: v1 -kind: Service -metadata: - name: coredns - namespace: kube-system - labels: - k8s-app: kube-dns - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "CoreDNS" - addonmanager.kubernetes.io/mode: Reconcile - annotations: - prometheus.io/port: "9153" - prometheus.io/scrape: "true" - createdby: 'kubekey' -spec: - selector: - k8s-app: kube-dns - clusterIP: {{ .ClusterIP }} - ports: - - name: dns - port: 53 - protocol: UDP - - name: dns-tcp - port: 53 - protocol: TCP - - name: metrics - port: 9153 - protocol: TCP - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: "coredns" - namespace: kube-system - labels: - k8s-app: "kube-dns" - addonmanager.kubernetes.io/mode: Reconcile - kubernetes.io/name: "coredns" -spec: - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 0 - maxSurge: 10% - selector: - matchLabels: - k8s-app: kube-dns - template: - metadata: - labels: - k8s-app: kube-dns - annotations: - createdby: 'kubekey' - spec: - securityContext: - seccompProfile: - type: RuntimeDefault - priorityClassName: system-cluster-critical - serviceAccountName: coredns - nodeSelector: - kubernetes.io/os: linux - tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule - - key: node-role.kubernetes.io/control-plane - effect: NoSchedule - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchLabels: - k8s-app: kube-dns - topologyKey: "kubernetes.io/hostname" - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - preference: - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: In - values: - - "" - containers: - - name: coredns - image: "{{ .CorednsImage }}" - imagePullPolicy: IfNotPresent - resources: - # TODO: Set memory limits when we've profiled the container for large - # clusters, then set request = limit to keep this container in - # guaranteed class. Currently, this container falls into the - # "burstable" category so the kubelet doesn't backoff from restarting it. - limits: - memory: 300Mi - requests: - cpu: 100m - memory: 70Mi - args: [ "-conf", "/etc/coredns/Corefile" ] - volumeMounts: - - name: config-volume - mountPath: /etc/coredns - ports: - - containerPort: 53 - name: dns - protocol: UDP - - containerPort: 53 - name: dns-tcp - protocol: TCP - - containerPort: 9153 - name: metrics - protocol: TCP - securityContext: - allowPrivilegeEscalation: false - capabilities: - add: - - NET_BIND_SERVICE - drop: - - all - readOnlyRootFilesystem: true - livenessProbe: - httpGet: - path: /health - port: 8080 - scheme: HTTP - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 10 - readinessProbe: - httpGet: - path: /ready - port: 8181 - scheme: HTTP - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 10 - dnsPolicy: Default - volumes: - - name: config-volume - configMap: - name: coredns - items: - - key: Corefile - path: Corefile -{{ if .DNSEtcHosts }} - - key: hosts - path: hosts -{{ end }} - - `))) -) diff --git a/cmd/kk/pkg/plugins/dns/templates/coredns_configmap.go b/cmd/kk/pkg/plugins/dns/templates/coredns_configmap.go deleted file mode 100644 index fde5f4c36..000000000 --- a/cmd/kk/pkg/plugins/dns/templates/coredns_configmap.go +++ /dev/null @@ -1,97 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" - "github.com/lithammer/dedent" - "text/template" -) - -var ( - CorednsConfigMap = template.Must(template.New("coredns-configmap.yaml").Funcs(utils.FuncMap).Parse( - dedent.Dedent(`--- -apiVersion: v1 -kind: ConfigMap -metadata: - name: coredns - namespace: kube-system - labels: - addonmanager.kubernetes.io/mode: EnsureExists -data: - Corefile: | -{{- if .ExternalZones }} -{{- range .ExternalZones }} -{{ range .Zones }}{{ . | indent 4 }} {{ end }}{ - log - errors -{{- if .Rewrite }} -{{- range .Rewrite }} - rewrite {{ . }} -{{- end }} -{{- end }} - forward .{{ range .Nameservers }} {{ . }}{{ end}} - loadbalance - cache {{ .Cache }} - reload -{{- if $.DNSEtcHosts }} - hosts /etc/coredns/hosts { - fallthrough - } -{{- end }} - } -{{- end }} -{{- end }} - .:53 { -{{- if .AdditionalConfigs }} -{{ .AdditionalConfigs | indent 8 }} -{{- end }} - errors - health { - lameduck 5s - } -{{- if .RewriteBlock }} -{{ .RewriteBlock | indent 8 }} -{{- end }} - ready - kubernetes {{ .ClusterDomain }} in-addr.arpa ip6.arpa { - pods insecure - fallthrough in-addr.arpa ip6.arpa - ttl 30 - } - prometheus :9153 - forward . {{ if .UpstreamDNSServers }}{{ range .UpstreamDNSServers }}{{ . }} {{ end }}{{else}}/etc/resolv.conf{{ end }} { - prefer_udp - max_concurrent 1000 - } - cache 30 - loop - reload - loadbalance -{{- if .DNSEtcHosts }} - hosts /etc/coredns/hosts { - fallthrough - } -{{- end }} - } -{{- if .DNSEtcHosts }} - hosts: | -{{ .DNSEtcHosts | indent 4 }} -{{- end }} - - `))) -) diff --git a/cmd/kk/pkg/plugins/dns/templates/nodelocaldns.go b/cmd/kk/pkg/plugins/dns/templates/nodelocaldns.go deleted file mode 100644 index 6d64ebdad..000000000 --- a/cmd/kk/pkg/plugins/dns/templates/nodelocaldns.go +++ /dev/null @@ -1,136 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var NodeLocalDNSService = template.Must(template.New("nodelocaldns.yaml").Parse( - dedent.Dedent(`--- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: nodelocaldns - namespace: kube-system - labels: - addonmanager.kubernetes.io/mode: Reconcile - ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: nodelocaldns - namespace: kube-system - labels: - k8s-app: kube-dns - addonmanager.kubernetes.io/mode: Reconcile -spec: - selector: - matchLabels: - k8s-app: nodelocaldns - template: - metadata: - labels: - k8s-app: nodelocaldns - annotations: - prometheus.io/scrape: 'true' - prometheus.io/port: '9253' - spec: - nodeSelector: - kubernetes.io/os: linux - priorityClassName: system-cluster-critical - serviceAccountName: nodelocaldns - hostNetwork: true - dnsPolicy: Default # Don't use cluster DNS. - tolerations: - - effect: NoSchedule - operator: "Exists" - - effect: NoExecute - operator: "Exists" - - key: "CriticalAddonsOnly" - operator: "Exists" - containers: - - name: node-cache - image: {{ .NodelocaldnsImage }} - resources: - limits: - memory: 200Mi - requests: - cpu: 100m - memory: 70Mi - args: [ "-localip", "169.254.25.10", "-conf", "/etc/coredns/Corefile", "-upstreamsvc", "coredns" ] - securityContext: - privileged: true - ports: - - containerPort: 53 - name: dns - protocol: UDP - - containerPort: 53 - name: dns-tcp - protocol: TCP - - containerPort: 9253 - name: metrics - protocol: TCP - livenessProbe: - httpGet: - host: 169.254.25.10 - path: /health - port: 9254 - scheme: HTTP - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 10 - readinessProbe: - httpGet: - host: 169.254.25.10 - path: /health - port: 9254 - scheme: HTTP - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 10 - volumeMounts: - - name: config-volume - mountPath: /etc/coredns - - name: xtables-lock - mountPath: /run/xtables.lock - volumes: - - name: config-volume - configMap: - name: nodelocaldns - items: - - key: Corefile - path: Corefile -{{- if .DNSEtcHosts }} - - key: hosts - path: hosts -{{ end }} - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: FileOrCreate - # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force - # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. - terminationGracePeriodSeconds: 0 - updateStrategy: - rollingUpdate: - maxUnavailable: 20% - type: RollingUpdate - - `))) diff --git a/cmd/kk/pkg/plugins/dns/templates/nodelocaldns_configmap.go b/cmd/kk/pkg/plugins/dns/templates/nodelocaldns_configmap.go deleted file mode 100644 index 556115007..000000000 --- a/cmd/kk/pkg/plugins/dns/templates/nodelocaldns_configmap.go +++ /dev/null @@ -1,116 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" - "github.com/lithammer/dedent" - "text/template" -) - -var NodeLocalDNSConfigMap = template.Must(template.New("nodelocaldns-configmap.yaml").Funcs(utils.FuncMap).Parse( - dedent.Dedent(`--- -apiVersion: v1 -kind: ConfigMap -metadata: - name: nodelocaldns - namespace: kube-system - labels: - addonmanager.kubernetes.io/mode: EnsureExists - -data: - Corefile: | -{{- if .ExternalZones }} -{{- range .ExternalZones }} -{{ range .Zones }}{{ . | indent 4 }} {{ end}} { - errors - cache {{ .Cache }} - reload -{{- if .Rewrite }} -{{- range .Rewrite }} - rewrite {{ . }} -{{- end }} -{{- end }} - loop - bind 169.254.25.10 - forward . {{ range .Nameservers }} {{ . }}{{ end }} - prometheus :9253 - log -{{- if $.DNSEtcHosts }} - hosts /etc/coredns/hosts { - fallthrough - } -{{- end }} - } -{{- end }} -{{- end }} - {{ .DNSDomain }}:53 { - errors - cache { - success 9984 30 - denial 9984 5 - } - reload - loop - bind 169.254.25.10 - forward . {{ .ForwardTarget }} { - force_tcp - } - prometheus :9253 - health 169.254.25.10:9254 - } - in-addr.arpa:53 { - errors - cache 30 - reload - loop - bind 169.254.25.10 - forward . {{ .ForwardTarget }} { - force_tcp - } - prometheus :9253 - } - ip6.arpa:53 { - errors - cache 30 - reload - loop - bind 169.254.25.10 - forward . {{ .ForwardTarget }} { - force_tcp - } - prometheus :9253 - } - .:53 { - errors - cache 30 - reload - loop - bind 169.254.25.10 - forward . /etc/resolv.conf - prometheus :9253 -{{- if .DNSEtcHosts }} - hosts /etc/coredns/hosts { - fallthrough - } -{{- end }} - } -{{- if .DNSEtcHosts }} - hosts: | -{{ .DNSEtcHosts | indent 4}} -{{- end }} -`))) diff --git a/cmd/kk/pkg/plugins/kata.go b/cmd/kk/pkg/plugins/kata.go deleted file mode 100644 index 3d0b47764..000000000 --- a/cmd/kk/pkg/plugins/kata.go +++ /dev/null @@ -1,206 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package plugins - -import ( - "path/filepath" - "text/template" - - "github.com/lithammer/dedent" - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" -) - -// Kata Containers is an open source community working to build a secure container runtime with lightweight virtual -// machines that feel and perform like containers, but provide stronger workload isolation using hardware virtualization -// technology as a second layer of defense. - -var ( - KataDeploy = template.Must(template.New("kata-deploy.yaml").Parse( - dedent.Dedent(`--- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kata-label-node - namespace: kube-system ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: node-labeler -rules: -- apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "patch"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: kata-label-node-rb -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: node-labeler -subjects: -- kind: ServiceAccount - name: kata-label-node - namespace: kube-system ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kata-deploy - namespace: kube-system -spec: - selector: - matchLabels: - name: kata-deploy - template: - metadata: - labels: - name: kata-deploy - spec: - serviceAccountName: kata-label-node - containers: - - name: kube-kata - image: {{ .KataDeployImage }} - imagePullPolicy: IfNotPresent - lifecycle: - preStop: - exec: - command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh cleanup"] - command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh install" ] - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - privileged: false - volumeMounts: - - name: crio-conf - mountPath: /etc/crio/ - - name: containerd-conf - mountPath: /etc/containerd/ - - name: kata-artifacts - mountPath: /opt/kata/ - - name: dbus - mountPath: /var/run/dbus - - name: systemd - mountPath: /run/systemd - - name: local-bin - mountPath: /usr/local/bin/ - volumes: - - name: crio-conf - hostPath: - path: /etc/crio/ - - name: containerd-conf - hostPath: - path: /etc/containerd/ - - name: kata-artifacts - hostPath: - path: /opt/kata/ - type: DirectoryOrCreate - - name: dbus - hostPath: - path: /var/run/dbus - - name: systemd - hostPath: - path: /run/systemd - - name: local-bin - hostPath: - path: /usr/local/bin/ - updateStrategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate ---- -kind: RuntimeClass -apiVersion: node.k8s.io/v1beta1 -metadata: - name: kata-qemu -handler: kata-qemu -overhead: - podFixed: - memory: "160Mi" - cpu: "250m" ---- -kind: RuntimeClass -apiVersion: node.k8s.io/v1beta1 -metadata: - name: kata-clh -handler: kata-clh -overhead: - podFixed: - memory: "130Mi" - cpu: "250m" ---- -kind: RuntimeClass -apiVersion: node.k8s.io/v1beta1 -metadata: - name: kata-fc -handler: kata-fc -overhead: - podFixed: - memory: "130Mi" - cpu: "250m" - `))) -) - -func DeployKataTasks(d *DeployPluginsModule) []task.Interface { - generateKataDeployManifests := &task.RemoteTask{ - Name: "GenerateKataDeployManifests", - Desc: "Generate kata-deploy manifests", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: &action.Template{ - Template: KataDeploy, - Data: util.Data{ - "KataDeployImage": images.GetImage(d.Runtime, d.KubeConf, "kata-deploy").ImageName(), - }, - Dst: filepath.Join(common.KubeAddonsDir, KataDeploy.Name()), - }, - Parallel: false, - } - - deployKata := &task.RemoteTask{ - Name: "ApplyKataDeployManifests", - Desc: "Apply kata-deploy manifests", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(ApplyKataDeployManifests), - } - - return []task.Interface{ - generateKataDeployManifests, - deployKata, - } -} - -type ApplyKataDeployManifests struct { - common.KubeAction -} - -func (a *ApplyKataDeployManifests) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/addons/kata-deploy.yaml", true); err != nil { - return errors.Wrap(errors.WithStack(err), "apply kata-deploy manifests failed") - } - return nil -} diff --git a/cmd/kk/pkg/plugins/modules.go b/cmd/kk/pkg/plugins/modules.go deleted file mode 100644 index f69800fcd..000000000 --- a/cmd/kk/pkg/plugins/modules.go +++ /dev/null @@ -1,35 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package plugins - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" -) - -type DeployPluginsModule struct { - common.KubeModule -} - -func (d *DeployPluginsModule) Init() { - d.Name = "DeployPluginsModule" - d.Desc = "Deploy plugins for cluster" - - if d.KubeConf.Cluster.Kubernetes.EnableKataDeploy() && (d.KubeConf.Cluster.Kubernetes.ContainerManager == common.Containerd || d.KubeConf.Cluster.Kubernetes.ContainerManager == common.Crio) { - d.Tasks = append(d.Tasks, DeployKataTasks(d)...) - } - - if d.KubeConf.Cluster.Kubernetes.EnableNodeFeatureDiscovery() { - d.Tasks = append(d.Tasks, DeployNodeFeatureDiscoveryTasks(d)...) - } -} diff --git a/cmd/kk/pkg/plugins/network/cilium-1.11.7.tgz b/cmd/kk/pkg/plugins/network/cilium-1.11.7.tgz deleted file mode 100644 index 7effa6f3f..000000000 Binary files a/cmd/kk/pkg/plugins/network/cilium-1.11.7.tgz and /dev/null differ diff --git a/cmd/kk/pkg/plugins/network/hybridnet-0.6.6.tgz b/cmd/kk/pkg/plugins/network/hybridnet-0.6.6.tgz deleted file mode 100644 index a85ac275d..000000000 Binary files a/cmd/kk/pkg/plugins/network/hybridnet-0.6.6.tgz and /dev/null differ diff --git a/cmd/kk/pkg/plugins/network/modules.go b/cmd/kk/pkg/plugins/network/modules.go deleted file mode 100644 index db28cf6a0..000000000 --- a/cmd/kk/pkg/plugins/network/modules.go +++ /dev/null @@ -1,386 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package network - -import ( - "path/filepath" - - versionutil "k8s.io/apimachinery/pkg/util/version" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/network/templates" -) - -type DeployNetworkPluginModule struct { - common.KubeModule -} - -func (d *DeployNetworkPluginModule) Init() { - d.Name = "DeployNetworkPluginModule" - d.Desc = "Deploy cluster network plugin" - - switch d.KubeConf.Cluster.Network.Plugin { - case common.Calico: - d.Tasks = deployCalico(d) - case common.Flannel: - d.Tasks = deployFlannel(d) - case common.Cilium: - d.Tasks = deployCilium(d) - case common.Kubeovn: - d.Tasks = deployKubeOVN(d) - case common.Hybridnet: - d.Tasks = deployHybridnet(d) - default: - return - } - if d.KubeConf.Cluster.Network.EnableMultusCNI() { - d.Tasks = append(d.Tasks, deployMultus(d)...) - } -} - -func deployMultus(d *DeployNetworkPluginModule) []task.Interface { - generateMultus := &task.RemoteTask{ - Name: "GenerateMultus", - Desc: "Generate multus cni", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &OldK8sVersion{Not: true}, - }, - Action: &action.Template{ - Template: templates.Multus, - Dst: filepath.Join(common.KubeConfigDir, templates.Multus.Name()), - Data: util.Data{ - "MultusImage": images.GetImage(d.Runtime, d.KubeConf, "multus").ImageName(), - }, - }, - Parallel: true, - } - deploy := &task.RemoteTask{ - Name: "DeployMultus", - Desc: "Deploy multus", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(DeployNetworkMultusPlugin), - Parallel: true, - Retry: 5, - } - return []task.Interface{ - generateMultus, - deploy, - } -} - -func deployCalico(d *DeployNetworkPluginModule) []task.Interface { - generateCalicoOld := &task.RemoteTask{ - Name: "GenerateCalico", - Desc: "Generate calico", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(OldK8sVersion), - }, - Action: &action.Template{ - Template: templates.CalicoOld, - Dst: filepath.Join(common.KubeConfigDir, templates.CalicoOld.Name()), - Data: util.Data{ - "KubePodsCIDR": d.KubeConf.Cluster.Network.KubePodsCIDR, - "CalicoCniImage": images.GetImage(d.Runtime, d.KubeConf, "calico-cni").ImageName(), - "CalicoNodeImage": images.GetImage(d.Runtime, d.KubeConf, "calico-node").ImageName(), - "CalicoFlexvolImage": images.GetImage(d.Runtime, d.KubeConf, "calico-flexvol").ImageName(), - "CalicoControllersImage": images.GetImage(d.Runtime, d.KubeConf, "calico-kube-controllers").ImageName(), - "TyphaEnabled": len(d.Runtime.GetHostsByRole(common.K8s)) > 50 || d.KubeConf.Cluster.Network.Calico.Typha(), - "VethMTU": d.KubeConf.Cluster.Network.Calico.VethMTU, - "NodeCidrMaskSize": d.KubeConf.Cluster.Kubernetes.NodeCidrMaskSize, - "IPIPMode": d.KubeConf.Cluster.Network.Calico.IPIPMode, - "VXLANMode": d.KubeConf.Cluster.Network.Calico.VXLANMode, - }, - }, - Parallel: true, - } - - generateCalicoNew := &task.RemoteTask{ - Name: "GenerateCalico", - Desc: "Generate calico", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &OldK8sVersion{Not: true}, - }, - Action: &action.Template{ - Template: templates.CalicoNew, - Dst: filepath.Join(common.KubeConfigDir, templates.CalicoNew.Name()), - Data: util.Data{ - "KubePodsCIDR": d.KubeConf.Cluster.Network.KubePodsCIDR, - "CalicoCniImage": images.GetImage(d.Runtime, d.KubeConf, "calico-cni").ImageName(), - "CalicoNodeImage": images.GetImage(d.Runtime, d.KubeConf, "calico-node").ImageName(), - "CalicoFlexvolImage": images.GetImage(d.Runtime, d.KubeConf, "calico-flexvol").ImageName(), - "CalicoControllersImage": images.GetImage(d.Runtime, d.KubeConf, "calico-kube-controllers").ImageName(), - "CalicoTyphaImage": images.GetImage(d.Runtime, d.KubeConf, "calico-typha").ImageName(), - "TyphaEnabled": len(d.Runtime.GetHostsByRole(common.K8s)) > 50 || d.KubeConf.Cluster.Network.Calico.Typha(), - "VethMTU": d.KubeConf.Cluster.Network.Calico.VethMTU, - "NodeCidrMaskSize": d.KubeConf.Cluster.Kubernetes.NodeCidrMaskSize, - "IPIPMode": d.KubeConf.Cluster.Network.Calico.IPIPMode, - "VXLANMode": d.KubeConf.Cluster.Network.Calico.VXLANMode, - "ConatinerManagerIsIsula": d.KubeConf.Cluster.Kubernetes.ContainerManager == "isula", - "IPV4POOLNATOUTGOING": d.KubeConf.Cluster.Network.Calico.EnableIPV4POOL_NAT_OUTGOING(), - "DefaultIPPOOL": d.KubeConf.Cluster.Network.Calico.EnableDefaultIPPOOL(), - }, - }, - Parallel: true, - } - - deploy := &task.RemoteTask{ - Name: "DeployCalico", - Desc: "Deploy calico", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(DeployNetworkPlugin), - Parallel: true, - Retry: 5, - } - - if K8sVersionAtLeast(d.KubeConf.Cluster.Kubernetes.Version, "v1.16.0") { - return []task.Interface{ - generateCalicoNew, - deploy, - } - } else { - return []task.Interface{ - generateCalicoOld, - deploy, - } - } -} - -func deployFlannel(d *DeployNetworkPluginModule) []task.Interface { - generateFlannelPSP := &task.RemoteTask{ - Name: "GenerateFlannel", - Desc: "Generate flannel", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: &action.Template{ - Template: templates.FlannelPSP, - Dst: filepath.Join(common.KubeConfigDir, templates.FlannelPSP.Name()), - Data: util.Data{ - "KubePodsCIDR": d.KubeConf.Cluster.Network.KubePodsCIDR, - "FlannelImage": images.GetImage(d.Runtime, d.KubeConf, "flannel").ImageName(), - "FlannelPluginImage": images.GetImage(d.Runtime, d.KubeConf, "flannel-cni-plugin").ImageName(), - "BackendMode": d.KubeConf.Cluster.Network.Flannel.BackendMode, - }, - }, - Parallel: true, - } - generateFlannelPS := &task.RemoteTask{ - Name: "GenerateFlannel", - Desc: "Generate flannel", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: &action.Template{ - Template: templates.FlannelPS, - Dst: filepath.Join(common.KubeConfigDir, templates.FlannelPS.Name()), - Data: util.Data{ - "KubePodsCIDR": d.KubeConf.Cluster.Network.KubePodsCIDR, - "FlannelImage": images.GetImage(d.Runtime, d.KubeConf, "flannel").ImageName(), - "FlannelPluginImage": images.GetImage(d.Runtime, d.KubeConf, "flannel-cni-plugin").ImageName(), - "BackendMode": d.KubeConf.Cluster.Network.Flannel.BackendMode, - }, - }, - Parallel: true, - } - - deploy := &task.RemoteTask{ - Name: "DeployFlannel", - Desc: "Deploy flannel", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(DeployNetworkPlugin), - Parallel: true, - Retry: 5, - } - - if K8sVersionAtLeast(d.KubeConf.Cluster.Kubernetes.Version, "v1.25.0") { - return []task.Interface{ - generateFlannelPS, - deploy, - } - } else { - return []task.Interface{ - generateFlannelPSP, - deploy, - } - } -} - -func deployCilium(d *DeployNetworkPluginModule) []task.Interface { - - releaseCiliumChart := &task.LocalTask{ - Name: "GenerateCiliumChart", - Desc: "Generate cilium chart", - Action: new(ReleaseCiliumChart), - } - - syncCiliumChart := &task.RemoteTask{ - Name: "SyncCiliumChart", - Desc: "Synchronize cilium chart", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(SyncCiliumChart), - Parallel: true, - Retry: 2, - } - - deploy := &task.RemoteTask{ - Name: "DeployCilium", - Desc: "Deploy cilium", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(DeployCilium), - Parallel: true, - Retry: 5, - } - - return []task.Interface{ - releaseCiliumChart, - syncCiliumChart, - deploy, - } -} - -func deployKubeOVN(d *DeployNetworkPluginModule) []task.Interface { - label := &task.RemoteTask{ - Name: "LabelNode", - Desc: "Label node", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(LabelNode), - Parallel: true, - } - - ssl := &task.RemoteTask{ - Name: "GenerateSSl", - Desc: "Generate ssl", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(EnableSSL), - }, - Action: new(GenerateSSL), - Parallel: true, - } - - generateKubeOVN := &task.RemoteTask{ - Name: "GenerateKubeOVN", - Desc: "Generate kube-ovn", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(GenerateKubeOVN), - Parallel: true, - } - - deploy := &task.RemoteTask{ - Name: "DeployKubeOVN", - Desc: "Deploy kube-ovn", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(DeployKubeovnPlugin), - Parallel: true, - Retry: 5, - } - - kubectlKo := &task.RemoteTask{ - Name: "GenerateKubectlKo", - Desc: "Generate kubectl-ko", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Action: &action.Template{ - Template: templates.KubectlKo, - Dst: filepath.Join(common.BinDir, templates.KubectlKo.Name()), - }, - Parallel: true, - } - - chmod := &task.RemoteTask{ - Name: "ChmodKubectlKo", - Desc: "Chmod kubectl-ko", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Action: new(ChmodKubectlKo), - Parallel: true, - } - - return []task.Interface{ - label, - ssl, - generateKubeOVN, - deploy, - kubectlKo, - chmod, - } -} - -func deployHybridnet(d *DeployNetworkPluginModule) []task.Interface { - - releaseHybridnetChart := &task.LocalTask{ - Name: "GenerateHybridnetChart", - Desc: "Generate hybridnet chart", - Action: new(ReleaseHybridnetChart), - } - - syncHybridnetChart := &task.RemoteTask{ - Name: "SyncHybridnetChart", - Desc: "Synchronize hybridnet chart", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(SyncHybridnetChart), - Parallel: true, - Retry: 2, - } - - deploy := &task.RemoteTask{ - Name: "DeployHybridnet", - Desc: "Deploy hybridnet", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(DeployHybridnet), - Parallel: true, - Retry: 5, - } - - return []task.Interface{ - releaseHybridnetChart, - syncHybridnetChart, - deploy, - } -} - -func K8sVersionAtLeast(version string, compare string) bool { - cmp, err := versionutil.MustParseSemantic(version).Compare(compare) - if err != nil { - logger.Log.Fatal("unknown kubernetes version") - } - // old version - if cmp == -1 { - return false - } - // new version - return true -} diff --git a/cmd/kk/pkg/plugins/network/prepares.go b/cmd/kk/pkg/plugins/network/prepares.go deleted file mode 100644 index eff351159..000000000 --- a/cmd/kk/pkg/plugins/network/prepares.go +++ /dev/null @@ -1,53 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package network - -import ( - versionutil "k8s.io/apimachinery/pkg/util/version" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type OldK8sVersion struct { - common.KubePrepare - Not bool -} - -func (o *OldK8sVersion) PreCheck(_ connector.Runtime) (bool, error) { - cmp, err := versionutil.MustParseSemantic(o.KubeConf.Cluster.Kubernetes.Version).Compare("v1.16.0") - if err != nil { - return false, err - } - // old version - if cmp == -1 { - return !o.Not, nil - } - // new version - return o.Not, nil -} - -type EnableSSL struct { - common.KubePrepare -} - -func (e *EnableSSL) PreCheck(_ connector.Runtime) (bool, error) { - if e.KubeConf.Cluster.Network.Kubeovn.EnableSSL { - return true, nil - } - return false, nil -} diff --git a/cmd/kk/pkg/plugins/network/tasks.go b/cmd/kk/pkg/plugins/network/tasks.go deleted file mode 100644 index cd3cf3419..000000000 --- a/cmd/kk/pkg/plugins/network/tasks.go +++ /dev/null @@ -1,427 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package network - -import ( - "embed" - "fmt" - "github.com/pkg/errors" - "io" - "os" - "path/filepath" - "strings" - "time" - - "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/network/templates" -) - -//go:embed cilium-1.11.7.tgz hybridnet-0.6.6.tgz - -var f embed.FS - -type ReleaseCiliumChart struct { - common.KubeAction -} - -func (r *ReleaseCiliumChart) Execute(runtime connector.Runtime) error { - fs, err := os.Create(fmt.Sprintf("%s/cilium.tgz", runtime.GetWorkDir())) - if err != nil { - return err - } - chartFile, err := f.Open("cilium-1.11.7.tgz") - if err != nil { - return err - } - defer chartFile.Close() - - _, err = io.Copy(fs, chartFile) - if err != nil { - return err - } - - return nil -} - -type SyncCiliumChart struct { - common.KubeAction -} - -func (s *SyncCiliumChart) Execute(runtime connector.Runtime) error { - src := filepath.Join(runtime.GetWorkDir(), "cilium.tgz") - dst := filepath.Join(common.TmpDir, "cilium.tgz") - if err := runtime.GetRunner().Scp(src, dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync cilium chart failed")) - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("mv %s/cilium.tgz /etc/kubernetes", common.TmpDir), true); err != nil { - return errors.Wrap(errors.WithStack(err), "sync cilium chart failed") - } - return nil -} - -type DeployCilium struct { - common.KubeAction -} - -func (d *DeployCilium) Execute(runtime connector.Runtime) error { - ciliumImage := images.GetImage(runtime, d.KubeConf, "cilium").ImageName() - ciliumOperatorImage := images.GetImage(runtime, d.KubeConf, "cilium-operator-generic").ImageName() - - cmd := fmt.Sprintf("/usr/local/bin/helm upgrade --install cilium /etc/kubernetes/cilium.tgz --namespace kube-system "+ - "--set operator.image.override=%s "+ - "--set operator.replicas=1 "+ - "--set image.override=%s "+ - "--set ipam.operator.clusterPoolIPv4PodCIDR=%s", ciliumOperatorImage, ciliumImage, d.KubeConf.Cluster.Network.KubePodsCIDR) - - if d.KubeConf.Cluster.Kubernetes.DisableKubeProxy { - cmd = fmt.Sprintf("%s --set kubeProxyReplacement=strict --set k8sServiceHost=%s --set k8sServicePort=%d", cmd, d.KubeConf.Cluster.ControlPlaneEndpoint.Address, d.KubeConf.Cluster.ControlPlaneEndpoint.Port) - } - - if _, err := runtime.GetRunner().SudoCmd(cmd, true); err != nil { - return errors.Wrap(errors.WithStack(err), "deploy cilium failed") - } - return nil -} - -type DeployNetworkPlugin struct { - common.KubeAction -} - -func (d *DeployNetworkPlugin) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl apply -f /etc/kubernetes/network-plugin.yaml --force", true); err != nil { - return errors.Wrap(errors.WithStack(err), "deploy network plugin failed") - } - return nil -} - -type DeployKubeovnPlugin struct { - common.KubeAction -} - -func (d *DeployKubeovnPlugin) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl apply -f /etc/kubernetes/kube-ovn-crd.yaml --force", true); err != nil { - return errors.Wrap(errors.WithStack(err), "deploy kube-ovn-crd.yaml failed") - } - if _, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl apply -f /etc/kubernetes/ovn.yaml --force", true); err != nil { - return errors.Wrap(errors.WithStack(err), "deploy ovn.yaml failed") - } - if _, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl apply -f /etc/kubernetes/kube-ovn.yaml --force", true); err != nil { - return errors.Wrap(errors.WithStack(err), "deploy kube-ovn.yaml failed") - } - return nil -} - -type DeployNetworkMultusPlugin struct { - common.KubeAction -} - -func (d *DeployNetworkMultusPlugin) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl apply -f /etc/kubernetes/multus-network-plugin.yaml --force", true); err != nil { - return errors.Wrap(errors.WithStack(err), "deploy multus network plugin failed") - } - return nil -} - -type LabelNode struct { - common.KubeAction -} - -func (l *LabelNode) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd( - fmt.Sprintf("/usr/local/bin/kubectl label no -l%s kube-ovn/role=master --overwrite", - l.KubeConf.Cluster.Network.Kubeovn.Label), - true); err != nil { - return errors.Wrap(errors.WithStack(err), "label kubeovn/role=master in master node failed") - } - return nil -} - -type GenerateSSL struct { - common.KubeAction -} - -func (g *GenerateSSL) Execute(runtime connector.Runtime) error { - if exist, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl get secret -n kube-system kube-ovn-tls --ignore-not-found", - true); err != nil { - return errors.Wrap(errors.WithStack(err), "find ovn secret failed") - } else if exist != "" { - return nil - } - - if _, err := runtime.GetRunner().SudoCmd( - fmt.Sprintf("docker run --rm -v %s:/etc/ovn %s bash generate-ssl.sh", - runtime.GetWorkDir(), images.GetImage(runtime, g.KubeConf, "kubeovn").ImageName()), - true); err != nil { - return errors.Wrap(errors.WithStack(err), "generate ovn secret failed") - } - - if _, err := runtime.GetRunner().SudoCmd( - fmt.Sprintf("/usr/local/bin/kubectl create secret generic -n kube-system kube-ovn-tls "+ - "--from-file=cacert=%s/cacert.pem "+ - "--from-file=cert=%s/ovn-cert.pem "+ - "--from-file=key=%s/ovn-privkey.pem", - runtime.GetWorkDir(), runtime.GetWorkDir(), runtime.GetWorkDir()), - true); err != nil { - return errors.Wrap(errors.WithStack(err), "create ovn secret failed") - } - - if _, err := runtime.GetRunner().SudoCmd( - fmt.Sprintf("rm -rf %s/cacert.pem %s/ovn-cert.pem %s/ovn-privkey.pem %s/ovn-req.pem", - runtime.GetWorkDir(), runtime.GetWorkDir(), runtime.GetWorkDir(), runtime.GetWorkDir()), - true); err != nil { - return errors.Wrap(errors.WithStack(err), "remove generated ovn secret file failed") - } - - return nil -} - -type GenerateKubeOVN struct { - common.KubeAction -} - -func (g *GenerateKubeOVN) Execute(runtime connector.Runtime) error { - address, err := runtime.GetRunner().Cmd( - "/usr/local/bin/kubectl get no -lkube-ovn/role=master --no-headers -o wide | awk '{print $6}' | tr \\\\n ','", - true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get kube-ovn label node address failed") - } - - count, err := runtime.GetRunner().Cmd( - fmt.Sprintf("/usr/local/bin/kubectl get no -l%s --no-headers -o wide | wc -l | sed 's/ //g'", - g.KubeConf.Cluster.Network.Kubeovn.Label), true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "count kube-ovn label nodes num failed") - } - - if count == "0" { - return fmt.Errorf("no node with label: %s", g.KubeConf.Cluster.Network.Kubeovn.Label) - } - - templateAction := action.Template{ - Template: templates.KubeOvnCrd, - Dst: filepath.Join(common.KubeConfigDir, templates.KubeOvnCrd.Name()), - } - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - - templateAction = action.Template{ - Template: templates.OVN, - Dst: filepath.Join(common.KubeConfigDir, templates.OVN.Name()), - Data: util.Data{ - "Address": address, - "Count": count, - "KubeovnImage": images.GetImage(runtime, g.KubeConf, "kubeovn").ImageName(), - "TunnelType": g.KubeConf.Cluster.Network.Kubeovn.TunnelType, - "DpdkMode": g.KubeConf.Cluster.Network.Kubeovn.Dpdk.DpdkMode, - "DpdkVersion": g.KubeConf.Cluster.Network.Kubeovn.Dpdk.DpdkVersion, - "OvnVersion": v1alpha2.DefaultKubeovnVersion, - "EnableSSL": g.KubeConf.Cluster.Network.Kubeovn.EnableSSL, - "HwOffload": g.KubeConf.Cluster.Network.Kubeovn.OvsOvn.HwOffload, - "SvcYamlIpfamilypolicy": g.KubeConf.Cluster.Network.Kubeovn.SvcYamlIpfamilypolicy, - }, - } - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - - templateAction = action.Template{ - Template: templates.KubeOvn, - Dst: filepath.Join(common.KubeConfigDir, templates.KubeOvn.Name()), - Data: util.Data{ - "Address": address, - "Count": count, - "KubeovnImage": images.GetImage(runtime, g.KubeConf, "kubeovn").ImageName(), - "PodCIDR": g.KubeConf.Cluster.Network.KubePodsCIDR, - "SvcCIDR": g.KubeConf.Cluster.Network.KubeServiceCIDR, - "JoinCIDR": g.KubeConf.Cluster.Network.Kubeovn.JoinCIDR, - "PodGateway": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnController.PodGateway, - "CheckGateway": g.KubeConf.Cluster.Network.Kubeovn.KubeovnCheckGateway(), - "LogicalGateway": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnController.LogicalGateway, - "PingExternalAddress": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnPinger.PingerExternalAddress, - "PingExternalDNS": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnPinger.PingerExternalDomain, - "NetworkType": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnController.NetworkType, - "TunnelType": g.KubeConf.Cluster.Network.Kubeovn.TunnelType, - "ExcludeIps": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnController.ExcludeIps, - "PodNicType": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnController.PodNicType, - "VlanID": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnController.VlanID, - "VlanInterfaceName": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnController.VlanInterfaceName, - "Iface": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnCni.Iface, - "EnableSSL": g.KubeConf.Cluster.Network.Kubeovn.EnableSSL, - "EnableMirror": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnCni.EnableMirror, - "EnableLB": g.KubeConf.Cluster.Network.Kubeovn.KubeovnEnableLB(), - "EnableNP": g.KubeConf.Cluster.Network.Kubeovn.KubeovnEnableNP(), - "EnableEipSnat": g.KubeConf.Cluster.Network.Kubeovn.KubeovnEnableEipSnat(), - "EnableExternalVPC": g.KubeConf.Cluster.Network.Kubeovn.KubeovnEnableExternalVPC(), - "SvcYamlIpfamilypolicy": g.KubeConf.Cluster.Network.Kubeovn.SvcYamlIpfamilypolicy, - "DpdkTunnelIface": g.KubeConf.Cluster.Network.Kubeovn.Dpdk.DpdkTunnelIface, - "CNIConfigPriority": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnCni.CNIConfigPriority, - "Modules": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnCni.Modules, - "RPMs": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnCni.RPMs, - }, - } - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - - return nil -} - -type ChmodKubectlKo struct { - common.KubeAction -} - -func (c *ChmodKubectlKo) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd( - fmt.Sprintf("chmod +x %s", filepath.Join(common.BinDir, templates.KubectlKo.Name())), false); err != nil { - return errors.Wrap(errors.WithStack(err), "chmod +x kubectl-ko failed") - } - return nil -} - -// ReleaseHybridnetChart is used to release hybridnet chart to local path -type ReleaseHybridnetChart struct { - common.KubeAction -} - -func (r *ReleaseHybridnetChart) Execute(runtime connector.Runtime) error { - fs, err := os.Create(fmt.Sprintf("%s/hybridnet.tgz", runtime.GetWorkDir())) - if err != nil { - return err - } - chartFile, err := f.Open("hybridnet-0.6.6.tgz") - if err != nil { - return err - } - defer chartFile.Close() - - _, err = io.Copy(fs, chartFile) - if err != nil { - return err - } - - return nil -} - -// SyncHybridnetChart is used to sync hybridnet chart to contronplane -type SyncHybridnetChart struct { - common.KubeAction -} - -func (s *SyncHybridnetChart) Execute(runtime connector.Runtime) error { - src := filepath.Join(runtime.GetWorkDir(), "hybridnet.tgz") - dst := filepath.Join(common.TmpDir, "hybridnet.tgz") - if err := runtime.GetRunner().Scp(src, dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync hybridnet chart failed")) - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("mv %s/hybridnet.tgz /etc/kubernetes", common.TmpDir), true); err != nil { - return errors.Wrap(errors.WithStack(err), "sync hybrident chart failed") - } - return nil -} - -type DeployHybridnet struct { - common.KubeAction -} - -func (d *DeployHybridnet) Execute(runtime connector.Runtime) error { - - cmd := fmt.Sprintf("/usr/local/bin/helm upgrade --install hybridnet /etc/kubernetes/hybridnet.tgz --namespace kube-system "+ - "--set images.hybridnet.image=%s/%s "+ - "--set images.hybridnet.tag=%s "+ - "--set images.registryURL=%s ", - images.GetImage(runtime, d.KubeConf, "hybridnet").ImageNamespace(), - images.GetImage(runtime, d.KubeConf, "hybridnet").Repo, - images.GetImage(runtime, d.KubeConf, "hybridnet").Tag, - images.GetImage(runtime, d.KubeConf, "hybridnet").ImageRegistryAddr(), - ) - - if d.KubeConf.Cluster.Network.Hybridnet.EnableInit() { - cmd = fmt.Sprintf("%s --set init.cidr=%s", cmd, d.KubeConf.Cluster.Network.KubePodsCIDR) - } else { - cmd = fmt.Sprintf("%s --set init=null", cmd) - } - - if !d.KubeConf.Cluster.Network.Hybridnet.NetworkPolicy() { - cmd = fmt.Sprintf("%s --set daemon.enableNetworkPolicy=false", cmd) - } - - if d.KubeConf.Cluster.Network.Hybridnet.DefaultNetworkType != "" { - cmd = fmt.Sprintf("%s --set defaultNetworkType=%s", cmd, d.KubeConf.Cluster.Network.Hybridnet.DefaultNetworkType) - } - - if d.KubeConf.Cluster.Network.Hybridnet.PreferBGPInterfaces != "" { - cmd = fmt.Sprintf("%s --set daemon.preferBGPInterfaces=%s", cmd, d.KubeConf.Cluster.Network.Hybridnet.PreferBGPInterfaces) - } - - if d.KubeConf.Cluster.Network.Hybridnet.PreferVlanInterfaces != "" { - cmd = fmt.Sprintf("%s --set daemon.preferVlanInterfaces=%s", cmd, d.KubeConf.Cluster.Network.Hybridnet.PreferVlanInterfaces) - } - - if d.KubeConf.Cluster.Network.Hybridnet.PreferVxlanInterfaces != "" { - cmd = fmt.Sprintf("%s --set daemon.preferVxlanInterfaces=%s", cmd, d.KubeConf.Cluster.Network.Hybridnet.PreferVxlanInterfaces) - } - - if _, err := runtime.GetRunner().SudoCmd(cmd, true); err != nil { - return errors.Wrap(errors.WithStack(err), "deploy hybridnet failed") - } - - if len(d.KubeConf.Cluster.Network.Hybridnet.Networks) != 0 { - templateAction := action.Template{ - Template: templates.HybridnetNetworks, - Dst: filepath.Join(common.KubeConfigDir, templates.HybridnetNetworks.Name()), - Data: util.Data{ - "Networks": d.KubeConf.Cluster.Network.Hybridnet.Networks, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - - for i := 0; i < 30; i++ { - fmt.Println("Waiting for hybridnet webhook running ... ", i+1) - time.Sleep(10 * time.Second) - output, _ := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl get pod -n kube-system -l app=hybridnet,component=webhook | grep Running", false) - if strings.Contains(output, "1/1") { - time.Sleep(50 * time.Second) - break - } - } - - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/hybridnet-networks.yaml", true); err != nil { - return errors.Wrap(errors.WithStack(err), "apply hybridnet networks failed") - } - } - return nil -} diff --git a/cmd/kk/pkg/plugins/network/templates/calico_v1.16+.go b/cmd/kk/pkg/plugins/network/templates/calico_v1.16+.go deleted file mode 100644 index d803b611b..000000000 --- a/cmd/kk/pkg/plugins/network/templates/calico_v1.16+.go +++ /dev/null @@ -1,5207 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var CalicoNew = template.Must(template.New("network-plugin.yaml").Parse( - dedent.Dedent(` ---- -# Source: calico/templates/calico-kube-controllers.yaml -# This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict - -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers -spec: - maxUnavailable: 1 - selector: - matchLabels: - k8s-app: calico-kube-controllers - -{{ if .TyphaEnabled }} ---- -# Source: calico/templates/calico-typha.yaml -# This manifest creates a Pod Disruption Budget for Typha to allow K8s Cluster Autoscaler to evict - -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: calico-typha - namespace: kube-system - labels: - k8s-app: calico-typha -spec: - maxUnavailable: 1 - selector: - matchLabels: - k8s-app: calico-typha - -{{ end }} ---- -# Source: calico/templates/calico-kube-controllers.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-kube-controllers - namespace: kube-system ---- -# Source: calico/templates/calico-node.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-node - namespace: kube-system ---- -# Source: calico/templates/calico-node.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-cni-plugin - namespace: kube-system ---- -# Source: calico/templates/calico-config.yaml -# This ConfigMap is used to configure a self-hosted Calico installation. -kind: ConfigMap -apiVersion: v1 -metadata: - name: calico-config - namespace: kube-system -data: - # You must set a non-zero value for Typha replicas below. - typha_service_name: {{ if .TyphaEnabled }}"calico-typha"{{ else }}"none"{{ end }} - # Configure the backend to use. - calico_backend: "bird" - - # Configure the MTU to use for workload interfaces and tunnels. - # By default, MTU is auto-detected, and explicitly setting this field should not be required. - # You can override auto-detection by providing a non-zero value. - veth_mtu: "{{ .VethMTU }}" - - # The CNI network configuration to install on each node. The special - # values in this config will be automatically populated. - cni_network_config: |- - { - "name": "k8s-pod-network", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "calico", - "log_level": "info", - "log_file_path": "/var/log/calico/cni/cni.log", - "datastore_type": "kubernetes", - "nodename": "__KUBERNETES_NODE_NAME__", - "mtu": __CNI_MTU__, - "ipam": { - "type": "calico-ipam" - }, - "policy": { - "type": "k8s" - }, - "kubernetes": { - "kubeconfig": "__KUBECONFIG_FILEPATH__" - } - }, - { - "type": "portmap", - "snat": true, - "capabilities": {"portMappings": true} - }, - { - "type": "bandwidth", - "capabilities": {"bandwidth": true} - } - ] - } ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: bgpconfigurations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: BGPConfiguration - listKind: BGPConfigurationList - plural: bgpconfigurations - singular: bgpconfiguration - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: BGPConfiguration contains the configuration for any BGP routing. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BGPConfigurationSpec contains the values of the BGP configuration. - properties: - asNumber: - description: 'ASNumber is the default AS number used by a node. [Default: - 64512]' - format: int32 - type: integer - bindMode: - description: BindMode indicates whether to listen for BGP connections - on all addresses (None) or only on the node's canonical IP address - Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen - for BGP connections on all addresses. - type: string - communities: - description: Communities is a list of BGP community values and their - arbitrary names for tagging routes. - items: - description: Community contains standard or large community value - and its name. - properties: - name: - description: Name given to community value. - type: string - value: - description: Value must be of format aa:nn or aa:nn:mm. - For standard community use aa:nn format, where aa and - nn are 16 bit number. For large community use aa:nn:mm - format, where aa, nn and mm are 32 bit number. Where, - aa is an AS Number, nn and mm are per-AS identifier. - pattern: ^(\d+):(\d+)$|^(\d+):(\d+):(\d+)$ - type: string - type: object - type: array - ignoredInterfaces: - description: IgnoredInterfaces indicates the network interfaces that - needs to be excluded when reading device routes. - items: - type: string - type: array - listenPort: - description: ListenPort is the port where BGP protocol should listen. - Defaults to 179 - maximum: 65535 - minimum: 1 - type: integer - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which logs - are sent to the stdout. [Default: INFO]' - type: string - nodeMeshMaxRestartTime: - description: Time to allow for software restart for node-to-mesh peerings. When - specified, this is configured as the graceful restart timeout. When - not specified, the BIRD default of 120s is used. This field can - only be set on the default BGPConfiguration instance and requires - that NodeMesh is enabled - type: string - nodeMeshPassword: - description: Optional BGP password for full node-to-mesh peerings. - This field can only be set on the default BGPConfiguration instance - and requires that NodeMesh is enabled - properties: - secretKeyRef: - description: Selects a key of a secret in the node pod's namespace. - properties: - key: - description: The key of the secret to select from. Must be - a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the Secret or its key must be - defined - type: boolean - required: - - key - type: object - type: object - nodeToNodeMeshEnabled: - description: 'NodeToNodeMeshEnabled sets whether full node to node - BGP mesh is enabled. [Default: true]' - type: boolean - prefixAdvertisements: - description: PrefixAdvertisements contains per-prefix advertisement - configuration. - items: - description: PrefixAdvertisement configures advertisement properties - for the specified CIDR. - properties: - cidr: - description: CIDR for which properties should be advertised. - type: string - communities: - description: Communities can be list of either community names - already defined in Specs.Communities or community value - of format aa:nn or aa:nn:mm. For standard community use - aa:nn format, where aa and nn are 16 bit number. For - large community use aa:nn:mm format, where aa, nn and - mm are 32 bit number. Where,aa is an AS Number, nn and - mm are per-AS identifier. - items: - type: string - type: array - type: object - type: array - serviceClusterIPs: - description: ServiceClusterIPs are the CIDR blocks from which service - cluster IPs are allocated. If specified, Calico will advertise these - blocks, as well as any cluster IPs within them. - items: - description: ServiceClusterIPBlock represents a single allowed ClusterIP - CIDR block. - properties: - cidr: - type: string - type: object - type: array - serviceExternalIPs: - description: ServiceExternalIPs are the CIDR blocks for Kubernetes - Service External IPs. Kubernetes Service ExternalIPs will only be - advertised if they are within one of these blocks. - items: - description: ServiceExternalIPBlock represents a single allowed - External IP CIDR block. - properties: - cidr: - type: string - type: object - type: array - serviceLoadBalancerIPs: - description: ServiceLoadBalancerIPs are the CIDR blocks for Kubernetes - Service LoadBalancer IPs. Kubernetes Service status.LoadBalancer.Ingress - IPs will only be advertised if they are within one of these blocks. - items: - description: ServiceLoadBalancerIPBlock represents a single allowed - LoadBalancer IP CIDR block. - properties: - cidr: - type: string - type: object - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: bgpfilters.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: BGPFilter - listKind: BGPFilterList - plural: bgpfilters - singular: bgpfilter - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BGPFilterSpec contains the IPv4 and IPv6 filter rules of - the BGP Filter. - properties: - exportV4: - description: The ordered set of IPv4 BGPFilter rules acting on exporting - routes to a peer. - items: - description: BGPFilterRuleV4 defines a BGP filter rule consisting - a single IPv4 CIDR block and a filter action for this CIDR. - properties: - action: - type: string - cidr: - type: string - matchOperator: - type: string - required: - - action - - cidr - - matchOperator - type: object - type: array - exportV6: - description: The ordered set of IPv6 BGPFilter rules acting on exporting - routes to a peer. - items: - description: BGPFilterRuleV6 defines a BGP filter rule consisting - a single IPv6 CIDR block and a filter action for this CIDR. - properties: - action: - type: string - cidr: - type: string - matchOperator: - type: string - required: - - action - - cidr - - matchOperator - type: object - type: array - importV4: - description: The ordered set of IPv4 BGPFilter rules acting on importing - routes from a peer. - items: - description: BGPFilterRuleV4 defines a BGP filter rule consisting - a single IPv4 CIDR block and a filter action for this CIDR. - properties: - action: - type: string - cidr: - type: string - matchOperator: - type: string - required: - - action - - cidr - - matchOperator - type: object - type: array - importV6: - description: The ordered set of IPv6 BGPFilter rules acting on importing - routes from a peer. - items: - description: BGPFilterRuleV6 defines a BGP filter rule consisting - a single IPv6 CIDR block and a filter action for this CIDR. - properties: - action: - type: string - cidr: - type: string - matchOperator: - type: string - required: - - action - - cidr - - matchOperator - type: object - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: bgppeers.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: BGPPeer - listKind: BGPPeerList - plural: bgppeers - singular: bgppeer - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BGPPeerSpec contains the specification for a BGPPeer resource. - properties: - asNumber: - description: The AS Number of the peer. - format: int32 - type: integer - filters: - description: The ordered set of BGPFilters applied on this BGP peer. - items: - type: string - type: array - keepOriginalNextHop: - description: Option to keep the original nexthop field when routes - are sent to a BGP Peer. Setting "true" configures the selected BGP - Peers node to use the "next hop keep;" instead of "next hop self;"(default) - in the specific branch of the Node on "bird.cfg". - type: boolean - maxRestartTime: - description: Time to allow for software restart. When specified, - this is configured as the graceful restart timeout. When not specified, - the BIRD default of 120s is used. - type: string - node: - description: The node name identifying the Calico node instance that - is targeted by this peer. If this is not set, and no nodeSelector - is specified, then this BGP peer selects all nodes in the cluster. - type: string - nodeSelector: - description: Selector for the nodes that should have this peering. When - this is set, the Node field must be empty. - type: string - numAllowedLocalASNumbers: - description: Maximum number of local AS numbers that are allowed in - the AS path for received routes. This removes BGP loop prevention - and should only be used if absolutely necesssary. - format: int32 - type: integer - password: - description: Optional BGP password for the peerings generated by this - BGPPeer resource. - properties: - secretKeyRef: - description: Selects a key of a secret in the node pod's namespace. - properties: - key: - description: The key of the secret to select from. Must be - a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the Secret or its key must be - defined - type: boolean - required: - - key - type: object - type: object - peerIP: - description: The IP address of the peer followed by an optional port - number to peer with. If port number is given, format should be []:port - or : for IPv4. If optional port number is not set, - and this peer IP and ASNumber belongs to a calico/node with ListenPort - set in BGPConfiguration, then we use that port to peer. - type: string - peerSelector: - description: Selector for the remote nodes to peer with. When this - is set, the PeerIP and ASNumber fields must be empty. For each - peering between the local node and selected remote nodes, we configure - an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified, - and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The - remote AS number comes from the remote node's NodeBGPSpec.ASNumber, - or the global default if that is not set. - type: string - reachableBy: - description: Add an exact, i.e. /32, static route toward peer IP in - order to prevent route flapping. ReachableBy contains the address - of the gateway which peer can be reached by. - type: string - sourceAddress: - description: Specifies whether and how to configure a source address - for the peerings generated by this BGPPeer resource. Default value - "UseNodeIP" means to configure the node IP as the source address. "None" - means not to configure a source address. - type: string - ttlSecurity: - description: TTLSecurity enables the generalized TTL security mechanism - (GTSM) which protects against spoofed packets by ignoring received - packets with a smaller than expected TTL value. The provided value - is the number of hops (edges) between the peers. - type: integer - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: blockaffinities.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: BlockAffinity - listKind: BlockAffinityList - plural: blockaffinities - singular: blockaffinity - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BlockAffinitySpec contains the specification for a BlockAffinity - resource. - properties: - cidr: - type: string - deleted: - description: Deleted indicates that this block affinity is being deleted. - This field is a string for compatibility with older releases that - mistakenly treat this field as a string. - type: string - node: - type: string - state: - type: string - required: - - cidr - - deleted - - node - - state - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: caliconodestatuses.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: CalicoNodeStatus - listKind: CalicoNodeStatusList - plural: caliconodestatuses - singular: caliconodestatus - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: CalicoNodeStatusSpec contains the specification for a CalicoNodeStatus - resource. - properties: - classes: - description: Classes declares the types of information to monitor - for this calico/node, and allows for selective status reporting - about certain subsets of information. - items: - type: string - type: array - node: - description: The node name identifies the Calico node instance for - node status. - type: string - updatePeriodSeconds: - description: UpdatePeriodSeconds is the period at which CalicoNodeStatus - should be updated. Set to 0 to disable CalicoNodeStatus refresh. - Maximum update period is one day. - format: int32 - type: integer - type: object - status: - description: CalicoNodeStatusStatus defines the observed state of CalicoNodeStatus. - No validation needed for status since it is updated by Calico. - properties: - agent: - description: Agent holds agent status on the node. - properties: - birdV4: - description: BIRDV4 represents the latest observed status of bird4. - properties: - lastBootTime: - description: LastBootTime holds the value of lastBootTime - from bird.ctl output. - type: string - lastReconfigurationTime: - description: LastReconfigurationTime holds the value of lastReconfigTime - from bird.ctl output. - type: string - routerID: - description: Router ID used by bird. - type: string - state: - description: The state of the BGP Daemon. - type: string - version: - description: Version of the BGP daemon - type: string - type: object - birdV6: - description: BIRDV6 represents the latest observed status of bird6. - properties: - lastBootTime: - description: LastBootTime holds the value of lastBootTime - from bird.ctl output. - type: string - lastReconfigurationTime: - description: LastReconfigurationTime holds the value of lastReconfigTime - from bird.ctl output. - type: string - routerID: - description: Router ID used by bird. - type: string - state: - description: The state of the BGP Daemon. - type: string - version: - description: Version of the BGP daemon - type: string - type: object - type: object - bgp: - description: BGP holds node BGP status. - properties: - numberEstablishedV4: - description: The total number of IPv4 established bgp sessions. - type: integer - numberEstablishedV6: - description: The total number of IPv6 established bgp sessions. - type: integer - numberNotEstablishedV4: - description: The total number of IPv4 non-established bgp sessions. - type: integer - numberNotEstablishedV6: - description: The total number of IPv6 non-established bgp sessions. - type: integer - peersV4: - description: PeersV4 represents IPv4 BGP peers status on the node. - items: - description: CalicoNodePeer contains the status of BGP peers - on the node. - properties: - peerIP: - description: IP address of the peer whose condition we are - reporting. - type: string - since: - description: Since the state or reason last changed. - type: string - state: - description: State is the BGP session state. - type: string - type: - description: Type indicates whether this peer is configured - via the node-to-node mesh, or via en explicit global or - per-node BGPPeer object. - type: string - type: object - type: array - peersV6: - description: PeersV6 represents IPv6 BGP peers status on the node. - items: - description: CalicoNodePeer contains the status of BGP peers - on the node. - properties: - peerIP: - description: IP address of the peer whose condition we are - reporting. - type: string - since: - description: Since the state or reason last changed. - type: string - state: - description: State is the BGP session state. - type: string - type: - description: Type indicates whether this peer is configured - via the node-to-node mesh, or via en explicit global or - per-node BGPPeer object. - type: string - type: object - type: array - required: - - numberEstablishedV4 - - numberEstablishedV6 - - numberNotEstablishedV4 - - numberNotEstablishedV6 - type: object - lastUpdated: - description: LastUpdated is a timestamp representing the server time - when CalicoNodeStatus object last updated. It is represented in - RFC3339 form and is in UTC. - format: date-time - nullable: true - type: string - routes: - description: Routes reports routes known to the Calico BGP daemon - on the node. - properties: - routesV4: - description: RoutesV4 represents IPv4 routes on the node. - items: - description: CalicoNodeRoute contains the status of BGP routes - on the node. - properties: - destination: - description: Destination of the route. - type: string - gateway: - description: Gateway for the destination. - type: string - interface: - description: Interface for the destination - type: string - learnedFrom: - description: LearnedFrom contains information regarding - where this route originated. - properties: - peerIP: - description: If sourceType is NodeMesh or BGPPeer, IP - address of the router that sent us this route. - type: string - sourceType: - description: Type of the source where a route is learned - from. - type: string - type: object - type: - description: Type indicates if the route is being used for - forwarding or not. - type: string - type: object - type: array - routesV6: - description: RoutesV6 represents IPv6 routes on the node. - items: - description: CalicoNodeRoute contains the status of BGP routes - on the node. - properties: - destination: - description: Destination of the route. - type: string - gateway: - description: Gateway for the destination. - type: string - interface: - description: Interface for the destination - type: string - learnedFrom: - description: LearnedFrom contains information regarding - where this route originated. - properties: - peerIP: - description: If sourceType is NodeMesh or BGPPeer, IP - address of the router that sent us this route. - type: string - sourceType: - description: Type of the source where a route is learned - from. - type: string - type: object - type: - description: Type indicates if the route is being used for - forwarding or not. - type: string - type: object - type: array - type: object - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: clusterinformations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: ClusterInformation - listKind: ClusterInformationList - plural: clusterinformations - singular: clusterinformation - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: ClusterInformation contains the cluster specific information. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: ClusterInformationSpec contains the values of describing - the cluster. - properties: - calicoVersion: - description: CalicoVersion is the version of Calico that the cluster - is running - type: string - clusterGUID: - description: ClusterGUID is the GUID of the cluster - type: string - clusterType: - description: ClusterType describes the type of the cluster - type: string - datastoreReady: - description: DatastoreReady is used during significant datastore migrations - to signal to components such as Felix that it should wait before - accessing the datastore. - type: boolean - variant: - description: Variant declares which variant of Calico should be active. - type: string - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: felixconfigurations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: FelixConfiguration - listKind: FelixConfigurationList - plural: felixconfigurations - singular: felixconfiguration - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: Felix Configuration contains the configuration for Felix. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: FelixConfigurationSpec contains the values of the Felix configuration. - properties: - allowIPIPPacketsFromWorkloads: - description: 'AllowIPIPPacketsFromWorkloads controls whether Felix - will add a rule to drop IPIP encapsulated traffic from workloads - [Default: false]' - type: boolean - allowVXLANPacketsFromWorkloads: - description: 'AllowVXLANPacketsFromWorkloads controls whether Felix - will add a rule to drop VXLAN encapsulated traffic from workloads - [Default: false]' - type: boolean - awsSrcDstCheck: - description: 'Set source-destination-check on AWS EC2 instances. Accepted - value must be one of "DoNothing", "Enable" or "Disable". [Default: - DoNothing]' - enum: - - DoNothing - - Enable - - Disable - type: string - bpfConnectTimeLoadBalancingEnabled: - description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, - controls whether Felix installs the connection-time load balancer. The - connect-time load balancer is required for the host to be able to - reach Kubernetes services and it improves the performance of pod-to-service - connections. The only reason to disable it is for debugging purposes. [Default: - true]' - type: boolean - bpfDSROptoutCIDRs: - description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded - from DSR. That is, clients in those CIDRs will accesses nodeports - as if BPFExternalServiceMode was set to Tunnel. - items: - type: string - type: array - bpfDataIfacePattern: - description: BPFDataIfacePattern is a regular expression that controls - which interfaces Felix should attach BPF programs to in order to - catch traffic to/from the network. This needs to match the interfaces - that Calico workload traffic flows over as well as any interfaces - that handle incoming traffic to nodeports and services from outside - the cluster. It should not match the workload interfaces (usually - named cali...). - type: string - bpfDisableUnprivileged: - description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled - sysctl to disable unprivileged use of BPF. This ensures that unprivileged - users cannot access Calico''s BPF maps and cannot insert their own - BPF programs to interfere with Calico''s. [Default: true]' - type: boolean - bpfEnabled: - description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. - [Default: false]' - type: boolean - bpfEnforceRPF: - description: 'BPFEnforceRPF enforce strict RPF on all host interfaces - with BPF programs regardless of what is the per-interfaces or global - setting. Possible values are Disabled, Strict or Loose. [Default: - Loose]' - type: string - bpfExtToServiceConnmark: - description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit - mark that is set on connections from an external client to a local - service. This mark allows us to control how packets of that connection - are routed within the host and how is routing interpreted by RPF - check. [Default: 0]' - type: integer - bpfExternalServiceMode: - description: 'BPFExternalServiceMode in BPF mode, controls how connections - from outside the cluster to services (node ports and cluster IPs) - are forwarded to remote workloads. If set to "Tunnel" then both - request and response traffic is tunneled to the remote node. If - set to "DSR", the request traffic is tunneled but the response traffic - is sent directly from the remote node. In "DSR" mode, the remote - node appears to use the IP of the ingress node; this requires a - permissive L2 network. [Default: Tunnel]' - type: string - bpfHostConntrackBypass: - description: 'BPFHostConntrackBypass Controls whether to bypass Linux - conntrack in BPF mode for workloads and services. [Default: true - - bypass Linux conntrack]' - type: boolean - bpfKubeProxyEndpointSlicesEnabled: - description: BPFKubeProxyEndpointSlicesEnabled in BPF mode, controls - whether Felix's embedded kube-proxy accepts EndpointSlices or not. - type: boolean - bpfKubeProxyIptablesCleanupEnabled: - description: 'BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF - mode, Felix will proactively clean up the upstream Kubernetes kube-proxy''s - iptables chains. Should only be enabled if kube-proxy is not running. [Default: - true]' - type: boolean - bpfKubeProxyMinSyncPeriod: - description: 'BPFKubeProxyMinSyncPeriod, in BPF mode, controls the - minimum time between updates to the dataplane for Felix''s embedded - kube-proxy. Lower values give reduced set-up latency. Higher values - reduce Felix CPU usage by batching up more work. [Default: 1s]' - type: string - bpfL3IfacePattern: - description: BPFL3IfacePattern is a regular expression that allows - to list tunnel devices like wireguard or vxlan (i.e., L3 devices) - in addition to BPFDataIfacePattern. That is, tunnel interfaces not - created by Calico, that Calico workload traffic flows over as well - as any interfaces that handle incoming traffic to nodeports and - services from outside the cluster. - type: string - bpfLogLevel: - description: 'BPFLogLevel controls the log level of the BPF programs - when in BPF dataplane mode. One of "Off", "Info", or "Debug". The - logs are emitted to the BPF trace pipe, accessible with the command - tc exec bpf debug. [Default: Off].' - type: string - bpfMapSizeConntrack: - description: 'BPFMapSizeConntrack sets the size for the conntrack - map. This map must be large enough to hold an entry for each active - connection. Warning: changing the size of the conntrack map can - cause disruption.' - type: integer - bpfMapSizeIPSets: - description: BPFMapSizeIPSets sets the size for ipsets map. The IP - sets map must be large enough to hold an entry for each endpoint - matched by every selector in the source/destination matches in network - policy. Selectors such as "all()" can result in large numbers of - entries (one entry per endpoint in that case). - type: integer - bpfMapSizeIfState: - description: BPFMapSizeIfState sets the size for ifstate map. The - ifstate map must be large enough to hold an entry for each device - (host + workloads) on a host. - type: integer - bpfMapSizeNATAffinity: - type: integer - bpfMapSizeNATBackend: - description: BPFMapSizeNATBackend sets the size for nat back end map. - This is the total number of endpoints. This is mostly more than - the size of the number of services. - type: integer - bpfMapSizeNATFrontend: - description: BPFMapSizeNATFrontend sets the size for nat front end - map. FrontendMap should be large enough to hold an entry for each - nodeport, external IP and each port in each service. - type: integer - bpfMapSizeRoute: - description: BPFMapSizeRoute sets the size for the routes map. The - routes map should be large enough to hold one entry per workload - and a handful of entries per host (enough to cover its own IPs and - tunnel IPs). - type: integer - bpfPSNATPorts: - anyOf: - - type: integer - - type: string - description: 'BPFPSNATPorts sets the range from which we randomly - pick a port if there is a source port collision. This should be - within the ephemeral range as defined by RFC 6056 (1024–65535) and - preferably outside the ephemeral ranges used by common operating - systems. Linux uses 32768–60999, while others mostly use the IANA - defined range 49152–65535. It is not necessarily a problem if this - range overlaps with the operating systems. Both ends of the range - are inclusive. [Default: 20000:29999]' - pattern: ^.* - x-kubernetes-int-or-string: true - bpfPolicyDebugEnabled: - description: BPFPolicyDebugEnabled when true, Felix records detailed - information about the BPF policy programs, which can be examined - with the calico-bpf command-line tool. - type: boolean - chainInsertMode: - description: 'ChainInsertMode controls whether Felix hooks the kernel''s - top-level iptables chains by inserting a rule at the top of the - chain or by appending a rule at the bottom. insert is the safe default - since it prevents Calico''s rules from being bypassed. If you switch - to append mode, be sure that the other rules in the chains signal - acceptance by falling through to the Calico rules, otherwise the - Calico policy will be bypassed. [Default: insert]' - type: string - dataplaneDriver: - description: DataplaneDriver filename of the external dataplane driver - to use. Only used if UseInternalDataplaneDriver is set to false. - type: string - dataplaneWatchdogTimeout: - description: "DataplaneWatchdogTimeout is the readiness/liveness timeout - used for Felix's (internal) dataplane driver. Increase this value - if you experience spurious non-ready or non-live events when Felix - is under heavy load. Decrease the value to get felix to report non-live - or non-ready more quickly. [Default: 90s] \n Deprecated: replaced - by the generic HealthTimeoutOverrides." - type: string - debugDisableLogDropping: - type: boolean - debugMemoryProfilePath: - type: string - debugSimulateCalcGraphHangAfter: - type: string - debugSimulateDataplaneHangAfter: - type: string - defaultEndpointToHostAction: - description: 'DefaultEndpointToHostAction controls what happens to - traffic that goes from a workload endpoint to the host itself (after - the traffic hits the endpoint egress policy). By default Calico - blocks traffic from workload endpoints to the host itself with an - iptables "DROP" action. If you want to allow some or all traffic - from endpoint to host, set this parameter to RETURN or ACCEPT. Use - RETURN if you have your own rules in the iptables "INPUT" chain; - Calico will insert its rules at the top of that chain, then "RETURN" - packets to the "INPUT" chain once it has completed processing workload - endpoint egress policy. Use ACCEPT to unconditionally accept packets - from workloads after processing workload endpoint egress policy. - [Default: Drop]' - type: string - deviceRouteProtocol: - description: This defines the route protocol added to programmed device - routes, by default this will be RTPROT_BOOT when left blank. - type: integer - deviceRouteSourceAddress: - description: This is the IPv4 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. - type: string - deviceRouteSourceAddressIPv6: - description: This is the IPv6 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. - type: string - disableConntrackInvalidCheck: - type: boolean - endpointReportingDelay: - type: string - endpointReportingEnabled: - type: boolean - externalNodesList: - description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes - which may source tunnel traffic and have the tunneled traffic be - accepted at calico nodes. - items: - type: string - type: array - failsafeInboundHostPorts: - description: 'FailsafeInboundHostPorts is a list of UDP/TCP ports - and CIDRs that Felix will allow incoming traffic to host endpoints - on irrespective of the security policy. This is useful to avoid - accidentally cutting off a host with incorrect configuration. For - back-compatibility, if the protocol is not specified, it defaults - to "tcp". If a CIDR is not specified, it will allow traffic from - all addresses. To disable all inbound host ports, use the value - none. The default value allows ssh access and DHCP. [Default: tcp:22, - udp:68, tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667]' - items: - description: ProtoPort is combination of protocol, port, and CIDR. - Protocol and port must be specified. - properties: - net: - type: string - port: - type: integer - protocol: - type: string - required: - - port - - protocol - type: object - type: array - failsafeOutboundHostPorts: - description: 'FailsafeOutboundHostPorts is a list of UDP/TCP ports - and CIDRs that Felix will allow outgoing traffic from host endpoints - to irrespective of the security policy. This is useful to avoid - accidentally cutting off a host with incorrect configuration. For - back-compatibility, if the protocol is not specified, it defaults - to "tcp". If a CIDR is not specified, it will allow traffic from - all addresses. To disable all outbound host ports, use the value - none. The default value opens etcd''s standard ports to ensure that - Felix does not get cut off from etcd as well as allowing DHCP and - DNS. [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, - tcp:6667, udp:53, udp:67]' - items: - description: ProtoPort is combination of protocol, port, and CIDR. - Protocol and port must be specified. - properties: - net: - type: string - port: - type: integer - protocol: - type: string - required: - - port - - protocol - type: object - type: array - featureDetectOverride: - description: FeatureDetectOverride is used to override feature detection - based on auto-detected platform capabilities. Values are specified - in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". "true" - or "false" will force the feature, empty or omitted values are auto-detected. - type: string - featureGates: - description: FeatureGates is used to enable or disable tech-preview - Calico features. Values are specified in a comma separated list - with no spaces, example; "BPFConnectTimeLoadBalancingWorkaround=enabled,XyZ=false". - This is used to enable features that are not fully production ready. - type: string - floatingIPs: - description: FloatingIPs configures whether or not Felix will program - non-OpenStack floating IP addresses. (OpenStack-derived floating - IPs are always programmed, regardless of this setting.) - enum: - - Enabled - - Disabled - type: string - genericXDPEnabled: - description: 'GenericXDPEnabled enables Generic XDP so network cards - that don''t support XDP offload or driver modes can use XDP. This - is not recommended since it doesn''t provide better performance - than iptables. [Default: false]' - type: boolean - healthEnabled: - type: boolean - healthHost: - type: string - healthPort: - type: integer - healthTimeoutOverrides: - description: HealthTimeoutOverrides allows the internal watchdog timeouts - of individual subcomponents to be overridden. This is useful for - working around "false positive" liveness timeouts that can occur - in particularly stressful workloads or if CPU is constrained. For - a list of active subcomponents, see Felix's logs. - items: - properties: - name: - type: string - timeout: - type: string - required: - - name - - timeout - type: object - type: array - interfaceExclude: - description: 'InterfaceExclude is a comma-separated list of interfaces - that Felix should exclude when monitoring for host endpoints. The - default value ensures that Felix ignores Kubernetes'' IPVS dummy - interface, which is used internally by kube-proxy. If you want to - exclude multiple interface names using a single value, the list - supports regular expressions. For regular expressions you must wrap - the value with ''/''. For example having values ''/^kube/,veth1'' - will exclude all interfaces that begin with ''kube'' and also the - interface ''veth1''. [Default: kube-ipvs0]' - type: string - interfacePrefix: - description: 'InterfacePrefix is the interface name prefix that identifies - workload endpoints and so distinguishes them from host endpoint - interfaces. Note: in environments other than bare metal, the orchestrators - configure this appropriately. For example our Kubernetes and Docker - integrations set the ''cali'' value, and our OpenStack integration - sets the ''tap'' value. [Default: cali]' - type: string - interfaceRefreshInterval: - description: InterfaceRefreshInterval is the period at which Felix - rescans local interfaces to verify their state. The rescan can be - disabled by setting the interval to 0. - type: string - ipipEnabled: - description: 'IPIPEnabled overrides whether Felix should configure - an IPIP interface on the host. Optional as Felix determines this - based on the existing IP pools. [Default: nil (unset)]' - type: boolean - ipipMTU: - description: 'IPIPMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' - type: integer - ipsetsRefreshInterval: - description: 'IpsetsRefreshInterval is the period at which Felix re-checks - all iptables state to ensure that no other process has accidentally - broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: - 90s]' - type: string - iptablesBackend: - description: IptablesBackend specifies which backend of iptables will - be used. The default is Auto. - type: string - iptablesFilterAllowAction: - type: string - iptablesFilterDenyAction: - description: IptablesFilterDenyAction controls what happens to traffic - that is denied by network policy. By default Calico blocks traffic - with an iptables "DROP" action. If you want to use "REJECT" action - instead you can configure it in here. - type: string - iptablesLockFilePath: - description: 'IptablesLockFilePath is the location of the iptables - lock file. You may need to change this if the lock file is not in - its standard location (for example if you have mapped it into Felix''s - container at a different path). [Default: /run/xtables.lock]' - type: string - iptablesLockProbeInterval: - description: 'IptablesLockProbeInterval is the time that Felix will - wait between attempts to acquire the iptables lock if it is not - available. Lower values make Felix more responsive when the lock - is contended, but use more CPU. [Default: 50ms]' - type: string - iptablesLockTimeout: - description: 'IptablesLockTimeout is the time that Felix will wait - for the iptables lock, or 0, to disable. To use this feature, Felix - must share the iptables lock file with all other processes that - also take the lock. When running Felix inside a container, this - requires the /run directory of the host to be mounted into the calico/node - or calico/felix container. [Default: 0s disabled]' - type: string - iptablesMangleAllowAction: - type: string - iptablesMarkMask: - description: 'IptablesMarkMask is the mask that Felix selects its - IPTables Mark bits from. Should be a 32 bit hexadecimal number with - at least 8 bits set, none of which clash with any other mark bits - in use on the system. [Default: 0xff000000]' - format: int32 - type: integer - iptablesNATOutgoingInterfaceFilter: - type: string - iptablesPostWriteCheckInterval: - description: 'IptablesPostWriteCheckInterval is the period after Felix - has done a write to the dataplane that it schedules an extra read - back in order to check the write was not clobbered by another process. - This should only occur if another application on the system doesn''t - respect the iptables lock. [Default: 1s]' - type: string - iptablesRefreshInterval: - description: 'IptablesRefreshInterval is the period at which Felix - re-checks the IP sets in the dataplane to ensure that no other process - has accidentally broken Calico''s rules. Set to 0 to disable IP - sets refresh. Note: the default for this value is lower than the - other refresh intervals as a workaround for a Linux kernel bug that - was fixed in kernel version 4.11. If you are using v4.11 or greater - you may want to set this to, a higher value to reduce Felix CPU - usage. [Default: 10s]' - type: string - ipv6Support: - description: IPv6Support controls whether Felix enables support for - IPv6 (if supported by the in-use dataplane). - type: boolean - kubeNodePortRanges: - description: 'KubeNodePortRanges holds list of port ranges used for - service node ports. Only used if felix detects kube-proxy running - in ipvs mode. Felix uses these ranges to separate host and workload - traffic. [Default: 30000:32767].' - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - logDebugFilenameRegex: - description: LogDebugFilenameRegex controls which source code files - have their Debug log output included in the logs. Only logs from - files with names that match the given regular expression are included. The - filter only applies to Debug level logs. - type: string - logFilePath: - description: 'LogFilePath is the full path to the Felix log. Set to - none to disable file logging. [Default: /var/log/calico/felix.log]' - type: string - logPrefix: - description: 'LogPrefix is the log prefix that Felix uses when rendering - LOG rules. [Default: calico-packet]' - type: string - logSeverityFile: - description: 'LogSeverityFile is the log severity above which logs - are sent to the log file. [Default: Info]' - type: string - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which logs - are sent to the stdout. [Default: Info]' - type: string - logSeveritySys: - description: 'LogSeveritySys is the log severity above which logs - are sent to the syslog. Set to None for no logging to syslog. [Default: - Info]' - type: string - maxIpsetSize: - type: integer - metadataAddr: - description: 'MetadataAddr is the IP address or domain name of the - server that can answer VM queries for cloud-init metadata. In OpenStack, - this corresponds to the machine running nova-api (or in Ubuntu, - nova-api-metadata). A value of none (case insensitive) means that - Felix should not set up any NAT rule for the metadata path. [Default: - 127.0.0.1]' - type: string - metadataPort: - description: 'MetadataPort is the port of the metadata server. This, - combined with global.MetadataAddr (if not ''None''), is used to - set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. - In most cases this should not need to be changed [Default: 8775].' - type: integer - mtuIfacePattern: - description: MTUIfacePattern is a regular expression that controls - which interfaces Felix should scan in order to calculate the host's - MTU. This should not match workload interfaces (usually named cali...). - type: string - natOutgoingAddress: - description: NATOutgoingAddress specifies an address to use when performing - source NAT for traffic in a natOutgoing pool that is leaving the - network. By default the address used is an address on the interface - the traffic is leaving on (ie it uses the iptables MASQUERADE target) - type: string - natPortRange: - anyOf: - - type: integer - - type: string - description: NATPortRange specifies the range of ports that is used - for port mapping when doing outgoing NAT. When unset the default - behavior of the network stack is used. - pattern: ^.* - x-kubernetes-int-or-string: true - netlinkTimeout: - type: string - openstackRegion: - description: 'OpenstackRegion is the name of the region that a particular - Felix belongs to. In a multi-region Calico/OpenStack deployment, - this must be configured somehow for each Felix (here in the datamodel, - or in felix.cfg or the environment on each compute node), and must - match the [calico] openstack_region value configured in neutron.conf - on each node. [Default: Empty]' - type: string - policySyncPathPrefix: - description: 'PolicySyncPathPrefix is used to by Felix to communicate - policy changes to external services, like Application layer policy. - [Default: Empty]' - type: string - prometheusGoMetricsEnabled: - description: 'PrometheusGoMetricsEnabled disables Go runtime metrics - collection, which the Prometheus client does by default, when set - to false. This reduces the number of metrics reported, reducing - Prometheus load. [Default: true]' - type: boolean - prometheusMetricsEnabled: - description: 'PrometheusMetricsEnabled enables the Prometheus metrics - server in Felix if set to true. [Default: false]' - type: boolean - prometheusMetricsHost: - description: 'PrometheusMetricsHost is the host that the Prometheus - metrics server should bind to. [Default: empty]' - type: string - prometheusMetricsPort: - description: 'PrometheusMetricsPort is the TCP port that the Prometheus - metrics server should bind to. [Default: 9091]' - type: integer - prometheusProcessMetricsEnabled: - description: 'PrometheusProcessMetricsEnabled disables process metrics - collection, which the Prometheus client does by default, when set - to false. This reduces the number of metrics reported, reducing - Prometheus load. [Default: true]' - type: boolean - prometheusWireGuardMetricsEnabled: - description: 'PrometheusWireGuardMetricsEnabled disables wireguard - metrics collection, which the Prometheus client does by default, - when set to false. This reduces the number of metrics reported, - reducing Prometheus load. [Default: true]' - type: boolean - removeExternalRoutes: - description: Whether or not to remove device routes that have not - been programmed by Felix. Disabling this will allow external applications - to also add device routes. This is enabled by default which means - we will remove externally added routes. - type: boolean - reportingInterval: - description: 'ReportingInterval is the interval at which Felix reports - its status into the datastore or 0 to disable. Must be non-zero - in OpenStack deployments. [Default: 30s]' - type: string - reportingTTL: - description: 'ReportingTTL is the time-to-live setting for process-wide - status reports. [Default: 90s]' - type: string - routeRefreshInterval: - description: 'RouteRefreshInterval is the period at which Felix re-checks - the routes in the dataplane to ensure that no other process has - accidentally broken Calico''s rules. Set to 0 to disable route refresh. - [Default: 90s]' - type: string - routeSource: - description: 'RouteSource configures where Felix gets its routing - information. - WorkloadIPs: use workload endpoints to construct - routes. - CalicoIPAM: the default - use IPAM data to construct routes.' - type: string - routeSyncDisabled: - description: RouteSyncDisabled will disable all operations performed - on the route table. Set to true to run in network-policy mode only. - type: boolean - routeTableRange: - description: Deprecated in favor of RouteTableRanges. Calico programs - additional Linux route tables for various purposes. RouteTableRange - specifies the indices of the route tables that Calico should use. - properties: - max: - type: integer - min: - type: integer - required: - - max - - min - type: object - routeTableRanges: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRanges specifies a set of table index ranges - that Calico should use. DeprecatesRouteTableRange, overrides RouteTableRange. - items: - properties: - max: - type: integer - min: - type: integer - required: - - max - - min - type: object - type: array - serviceLoopPrevention: - description: 'When service IP advertisement is enabled, prevent routing - loops to service IPs that are not in use, by dropping or rejecting - packets that do not get DNAT''d by kube-proxy. Unless set to "Disabled", - in which case such routing loops continue to be allowed. [Default: - Drop]' - type: string - sidecarAccelerationEnabled: - description: 'SidecarAccelerationEnabled enables experimental sidecar - acceleration [Default: false]' - type: boolean - usageReportingEnabled: - description: 'UsageReportingEnabled reports anonymous Calico version - number and cluster size to projectcalico.org. Logs warnings returned - by the usage server. For example, if a significant security vulnerability - has been discovered in the version of Calico being used. [Default: - true]' - type: boolean - usageReportingInitialDelay: - description: 'UsageReportingInitialDelay controls the minimum delay - before Felix makes a report. [Default: 300s]' - type: string - usageReportingInterval: - description: 'UsageReportingInterval controls the interval at which - Felix makes reports. [Default: 86400s]' - type: string - useInternalDataplaneDriver: - description: UseInternalDataplaneDriver, if true, Felix will use its - internal dataplane programming logic. If false, it will launch - an external dataplane driver and communicate with it over protobuf. - type: boolean - vxlanEnabled: - description: 'VXLANEnabled overrides whether Felix should create the - VXLAN tunnel device for IPv4 VXLAN networking. Optional as Felix - determines this based on the existing IP pools. [Default: nil (unset)]' - type: boolean - vxlanMTU: - description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel - device. See Configuring MTU [Default: 1410]' - type: integer - vxlanMTUV6: - description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel - device. See Configuring MTU [Default: 1390]' - type: integer - vxlanPort: - type: integer - vxlanVNI: - type: integer - wireguardEnabled: - description: 'WireguardEnabled controls whether Wireguard is enabled - for IPv4 (encapsulating IPv4 traffic over an IPv4 underlay network). - [Default: false]' - type: boolean - wireguardEnabledV6: - description: 'WireguardEnabledV6 controls whether Wireguard is enabled - for IPv6 (encapsulating IPv6 traffic over an IPv6 underlay network). - [Default: false]' - type: boolean - wireguardHostEncryptionEnabled: - description: 'WireguardHostEncryptionEnabled controls whether Wireguard - host-to-host encryption is enabled. [Default: false]' - type: boolean - wireguardInterfaceName: - description: 'WireguardInterfaceName specifies the name to use for - the IPv4 Wireguard interface. [Default: wireguard.cali]' - type: string - wireguardInterfaceNameV6: - description: 'WireguardInterfaceNameV6 specifies the name to use for - the IPv6 Wireguard interface. [Default: wg-v6.cali]' - type: string - wireguardKeepAlive: - description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive - option. Set 0 to disable. [Default: 0]' - type: string - wireguardListeningPort: - description: 'WireguardListeningPort controls the listening port used - by IPv4 Wireguard. [Default: 51820]' - type: integer - wireguardListeningPortV6: - description: 'WireguardListeningPortV6 controls the listening port - used by IPv6 Wireguard. [Default: 51821]' - type: integer - wireguardMTU: - description: 'WireguardMTU controls the MTU on the IPv4 Wireguard - interface. See Configuring MTU [Default: 1440]' - type: integer - wireguardMTUV6: - description: 'WireguardMTUV6 controls the MTU on the IPv6 Wireguard - interface. See Configuring MTU [Default: 1420]' - type: integer - wireguardRoutingRulePriority: - description: 'WireguardRoutingRulePriority controls the priority value - to use for the Wireguard routing rule. [Default: 99]' - type: integer - workloadSourceSpoofing: - description: WorkloadSourceSpoofing controls whether pods can use - the allowedSourcePrefixes annotation to send traffic with a source - IP address that is not theirs. This is disabled by default. When - set to "Any", pods can request any prefix. - type: string - xdpEnabled: - description: 'XDPEnabled enables XDP acceleration for suitable untracked - incoming deny rules. [Default: true]' - type: boolean - xdpRefreshInterval: - description: 'XDPRefreshInterval is the period at which Felix re-checks - all XDP state to ensure that no other process has accidentally broken - Calico''s BPF maps or attached programs. Set to 0 to disable XDP - refresh. [Default: 90s]' - type: string - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: globalnetworkpolicies.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: GlobalNetworkPolicy - listKind: GlobalNetworkPolicyList - plural: globalnetworkpolicies - singular: globalnetworkpolicy - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - applyOnForward: - description: ApplyOnForward indicates to apply the rules in this policy - on forward traffic. - type: boolean - doNotTrack: - description: DoNotTrack indicates whether packets matched by the rules - in this policy should go through the data plane's connection tracking, - such as Linux conntrack. If True, the rules in this policy are - applied before any data plane connection tracking, and packets allowed - by this policy are marked as not to be tracked. - type: boolean - egress: - description: The ordered set of egress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with \"Not\". All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, global() - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a exact or a prefix match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, global() - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - required: - - action - type: object - type: array - ingress: - description: The ordered set of ingress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with \"Not\". All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, global() - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a exact or a prefix match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, global() - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - required: - - action - type: object - type: array - namespaceSelector: - description: NamespaceSelector is an optional field for an expression - used to select a pod based on namespaces. - type: string - order: - description: Order is an optional field that specifies the order in - which the policy is applied. Policies with higher "order" are applied - after those with lower order. If the order is omitted, it may be - considered to be "infinite" - i.e. the policy will be applied last. Policies - with identical order will be applied in alphanumerical order based - on the Policy "Name". - type: number - preDNAT: - description: PreDNAT indicates to apply the rules in this policy before - any DNAT. - type: boolean - selector: - description: "The selector is an expression used to pick pick out - the endpoints that the policy should be applied to. \n Selector - expressions follow this syntax: \n \tlabel == \"string_literal\" - \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" - \ -> not equal; also matches if label is not present \tlabel in - { \"a\", \"b\", \"c\", ... } -> true if the value of label X is - one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", - ... } -> true if the value of label X is not one of \"a\", \"b\", - \"c\" \thas(label_name) -> True if that label is present \t! expr - -> negation of expr \texpr && expr -> Short-circuit and \texpr - || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() - or the empty selector -> matches all endpoints. \n Label names are - allowed to contain alphanumerics, -, _ and /. String literals are - more permissive but they do not support escape characters. \n Examples - (with made-up labels): \n \ttype == \"webserver\" && deployment - == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != - \"dev\" \t! has(label_name)" - type: string - serviceAccountSelector: - description: ServiceAccountSelector is an optional field for an expression - used to select a pod based on service accounts. - type: string - types: - description: "Types indicates whether this policy applies to ingress, - or to egress, or to both. When not explicitly specified (and so - the value on creation is empty or nil), Calico defaults Types according - to what Ingress and Egress rules are present in the policy. The - default is: \n - [ PolicyTypeIngress ], if there are no Egress rules - (including the case where there are also no Ingress rules) \n - - [ PolicyTypeEgress ], if there are Egress rules but no Ingress - rules \n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are - both Ingress and Egress rules. \n When the policy is read back again, - Types will always be one of these values, never empty or nil." - items: - description: PolicyType enumerates the possible values of the PolicySpec - Types field. - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: globalnetworksets.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: GlobalNetworkSet - listKind: GlobalNetworkSetList - plural: globalnetworksets - singular: globalnetworkset - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: GlobalNetworkSet contains a set of arbitrary IP sub-networks/CIDRs - that share labels to allow rules to refer to them via selectors. The labels - of GlobalNetworkSet are not namespaced. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: GlobalNetworkSetSpec contains the specification for a NetworkSet - resource. - properties: - nets: - description: The list of IP networks that belong to this set. - items: - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: hostendpoints.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: HostEndpoint - listKind: HostEndpointList - plural: hostendpoints - singular: hostendpoint - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: HostEndpointSpec contains the specification for a HostEndpoint - resource. - properties: - expectedIPs: - description: "The expected IP addresses (IPv4 and IPv6) of the endpoint. - If \"InterfaceName\" is not present, Calico will look for an interface - matching any of the IPs in the list and apply policy to that. Note: - \tWhen using the selector match criteria in an ingress or egress - security Policy \tor Profile, Calico converts the selector into - a set of IP addresses. For host \tendpoints, the ExpectedIPs field - is used for that purpose. (If only the interface \tname is specified, - Calico does not learn the IPs of the interface for use in match - \tcriteria.)" - items: - type: string - type: array - interfaceName: - description: "Either \"*\", or the name of a specific Linux interface - to apply policy to; or empty. \"*\" indicates that this HostEndpoint - governs all traffic to, from or through the default network namespace - of the host named by the \"Node\" field; entering and leaving that - namespace via any interface, including those from/to non-host-networked - local workloads. \n If InterfaceName is not \"*\", this HostEndpoint - only governs traffic that enters or leaves the host through the - specific interface named by InterfaceName, or - when InterfaceName - is empty - through the specific interface that has one of the IPs - in ExpectedIPs. Therefore, when InterfaceName is empty, at least - one expected IP must be specified. Only external interfaces (such - as \"eth0\") are supported here; it isn't possible for a HostEndpoint - to protect traffic through a specific local workload interface. - \n Note: Only some kinds of policy are implemented for \"*\" HostEndpoints; - initially just pre-DNAT policy. Please check Calico documentation - for the latest position." - type: string - node: - description: The node name identifying the Calico node instance. - type: string - ports: - description: Ports contains the endpoint's named ports, which may - be referenced in security policy rules. - items: - properties: - name: - type: string - port: - type: integer - protocol: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - required: - - name - - port - - protocol - type: object - type: array - profiles: - description: A list of identifiers of security Profile objects that - apply to this endpoint. Each profile is applied in the order that - they appear in this list. Profile rules are applied after the selector-based - security policy. - items: - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: ipamblocks.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPAMBlock - listKind: IPAMBlockList - plural: ipamblocks - singular: ipamblock - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPAMBlockSpec contains the specification for an IPAMBlock - resource. - properties: - affinity: - description: Affinity of the block, if this block has one. If set, - it will be of the form "host:". If not set, this block - is not affine to a host. - type: string - allocations: - description: Array of allocations in-use within this block. nil entries - mean the allocation is free. For non-nil entries at index i, the - index is the ordinal of the allocation within this block and the - value is the index of the associated attributes in the Attributes - array. - items: - type: integer - # TODO: This nullable is manually added in. We should update controller-gen - # to handle []*int properly itself. - nullable: true - type: array - attributes: - description: Attributes is an array of arbitrary metadata associated - with allocations in the block. To find attributes for a given allocation, - use the value of the allocation's entry in the Allocations array - as the index of the element in this array. - items: - properties: - handle_id: - type: string - secondary: - additionalProperties: - type: string - type: object - type: object - type: array - cidr: - description: The block's CIDR. - type: string - deleted: - description: Deleted is an internal boolean used to workaround a limitation - in the Kubernetes API whereby deletion will not return a conflict - error if the block has been updated. It should not be set manually. - type: boolean - sequenceNumber: - default: 0 - description: We store a sequence number that is updated each time - the block is written. Each allocation will also store the sequence - number of the block at the time of its creation. When releasing - an IP, passing the sequence number associated with the allocation - allows us to protect against a race condition and ensure the IP - hasn't been released and re-allocated since the release request. - format: int64 - type: integer - sequenceNumberForAllocation: - additionalProperties: - format: int64 - type: integer - description: Map of allocated ordinal within the block to sequence - number of the block at the time of allocation. Kubernetes does not - allow numerical keys for maps, so the key is cast to a string. - type: object - strictAffinity: - description: StrictAffinity on the IPAMBlock is deprecated and no - longer used by the code. Use IPAMConfig StrictAffinity instead. - type: boolean - unallocated: - description: Unallocated is an ordered list of allocations which are - free in the block. - items: - type: integer - type: array - required: - - allocations - - attributes - - cidr - - strictAffinity - - unallocated - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: ipamconfigs.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPAMConfig - listKind: IPAMConfigList - plural: ipamconfigs - singular: ipamconfig - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPAMConfigSpec contains the specification for an IPAMConfig - resource. - properties: - autoAllocateBlocks: - type: boolean - maxBlocksPerHost: - description: MaxBlocksPerHost, if non-zero, is the max number of blocks - that can be affine to each host. - maximum: 2147483647 - minimum: 0 - type: integer - strictAffinity: - type: boolean - required: - - autoAllocateBlocks - - strictAffinity - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: ipamhandles.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPAMHandle - listKind: IPAMHandleList - plural: ipamhandles - singular: ipamhandle - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPAMHandleSpec contains the specification for an IPAMHandle - resource. - properties: - block: - additionalProperties: - type: integer - type: object - deleted: - type: boolean - handleID: - type: string - required: - - block - - handleID - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: ippools.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPPool - listKind: IPPoolList - plural: ippools - singular: ippool - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPPoolSpec contains the specification for an IPPool resource. - properties: - allowedUses: - description: AllowedUse controls what the IP pool will be used for. If - not specified or empty, defaults to ["Tunnel", "Workload"] for back-compatibility - items: - type: string - type: array - blockSize: - description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 122 for IPv6. - type: integer - cidr: - description: The pool CIDR. - type: string - disableBGPExport: - description: 'Disable exporting routes from this IP Pool''s CIDR over - BGP. [Default: false]' - type: boolean - disabled: - description: When disabled is true, Calico IPAM will not assign addresses - from this pool. - type: boolean - ipip: - description: 'Deprecated: this field is only used for APIv1 backwards - compatibility. Setting this field is not allowed, this field is - for internal use only.' - properties: - enabled: - description: When enabled is true, ipip tunneling will be used - to deliver packets to destinations within this pool. - type: boolean - mode: - description: The IPIP mode. This can be one of "always" or "cross-subnet". A - mode of "always" will also use IPIP tunneling for routing to - destination IP addresses within this pool. A mode of "cross-subnet" - will only use IPIP tunneling when the destination node is on - a different subnet to the originating node. The default value - (if not specified) is "always". - type: string - type: object - ipipMode: - description: Contains configuration for IPIP tunneling for this pool. - If not specified, then this is defaulted to "Never" (i.e. IPIP tunneling - is disabled). - type: string - nat-outgoing: - description: 'Deprecated: this field is only used for APIv1 backwards - compatibility. Setting this field is not allowed, this field is - for internal use only.' - type: boolean - natOutgoing: - description: When natOutgoing is true, packets sent from Calico networked - containers in this pool to destinations outside of this pool will - be masqueraded. - type: boolean - nodeSelector: - description: Allows IPPool to allocate for a specific node by label - selector. - type: string - vxlanMode: - description: Contains configuration for VXLAN tunneling for this pool. - If not specified, then this is defaulted to "Never" (i.e. VXLAN - tunneling is disabled). - type: string - required: - - cidr - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: ipreservations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPReservation - listKind: IPReservationList - plural: ipreservations - singular: ipreservation - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPReservationSpec contains the specification for an IPReservation - resource. - properties: - reservedCIDRs: - description: ReservedCIDRs is a list of CIDRs and/or IP addresses - that Calico IPAM will exclude from new allocations. - items: - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: kubecontrollersconfigurations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: KubeControllersConfiguration - listKind: KubeControllersConfigurationList - plural: kubecontrollersconfigurations - singular: kubecontrollersconfiguration - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: KubeControllersConfigurationSpec contains the values of the - Kubernetes controllers configuration. - properties: - controllers: - description: Controllers enables and configures individual Kubernetes - controllers - properties: - namespace: - description: Namespace enables and configures the namespace controller. - Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - node: - description: Node enables and configures the node controller. - Enabled by default, set to nil to disable. - properties: - hostEndpoint: - description: HostEndpoint controls syncing nodes to host endpoints. - Disabled by default, set to nil to disable. - properties: - autoCreate: - description: 'AutoCreate enables automatic creation of - host endpoints for every node. [Default: Disabled]' - type: string - type: object - leakGracePeriod: - description: 'LeakGracePeriod is the period used by the controller - to determine if an IP address has been leaked. Set to 0 - to disable IP garbage collection. [Default: 15m]' - type: string - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - syncLabels: - description: 'SyncLabels controls whether to copy Kubernetes - node labels to Calico nodes. [Default: Enabled]' - type: string - type: object - policy: - description: Policy enables and configures the policy controller. - Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - serviceAccount: - description: ServiceAccount enables and configures the service - account controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - workloadEndpoint: - description: WorkloadEndpoint enables and configures the workload - endpoint controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - type: object - debugProfilePort: - description: DebugProfilePort configures the port to serve memory - and cpu profiles on. If not specified, profiling is disabled. - format: int32 - type: integer - etcdV3CompactionPeriod: - description: 'EtcdV3CompactionPeriod is the period between etcdv3 - compaction requests. Set to 0 to disable. [Default: 10m]' - type: string - healthChecks: - description: 'HealthChecks enables or disables support for health - checks [Default: Enabled]' - type: string - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which logs - are sent to the stdout. [Default: Info]' - type: string - prometheusMetricsPort: - description: 'PrometheusMetricsPort is the TCP port that the Prometheus - metrics server should bind to. Set to 0 to disable. [Default: 9094]' - type: integer - required: - - controllers - type: object - status: - description: KubeControllersConfigurationStatus represents the status - of the configuration. It's useful for admins to be able to see the actual - config that was applied, which can be modified by environment variables - on the kube-controllers process. - properties: - environmentVars: - additionalProperties: - type: string - description: EnvironmentVars contains the environment variables on - the kube-controllers that influenced the RunningConfig. - type: object - runningConfig: - description: RunningConfig contains the effective config that is running - in the kube-controllers pod, after merging the API resource with - any environment variables. - properties: - controllers: - description: Controllers enables and configures individual Kubernetes - controllers - properties: - namespace: - description: Namespace enables and configures the namespace - controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - node: - description: Node enables and configures the node controller. - Enabled by default, set to nil to disable. - properties: - hostEndpoint: - description: HostEndpoint controls syncing nodes to host - endpoints. Disabled by default, set to nil to disable. - properties: - autoCreate: - description: 'AutoCreate enables automatic creation - of host endpoints for every node. [Default: Disabled]' - type: string - type: object - leakGracePeriod: - description: 'LeakGracePeriod is the period used by the - controller to determine if an IP address has been leaked. - Set to 0 to disable IP garbage collection. [Default: - 15m]' - type: string - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - syncLabels: - description: 'SyncLabels controls whether to copy Kubernetes - node labels to Calico nodes. [Default: Enabled]' - type: string - type: object - policy: - description: Policy enables and configures the policy controller. - Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - serviceAccount: - description: ServiceAccount enables and configures the service - account controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - workloadEndpoint: - description: WorkloadEndpoint enables and configures the workload - endpoint controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - type: object - debugProfilePort: - description: DebugProfilePort configures the port to serve memory - and cpu profiles on. If not specified, profiling is disabled. - format: int32 - type: integer - etcdV3CompactionPeriod: - description: 'EtcdV3CompactionPeriod is the period between etcdv3 - compaction requests. Set to 0 to disable. [Default: 10m]' - type: string - healthChecks: - description: 'HealthChecks enables or disables support for health - checks [Default: Enabled]' - type: string - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which - logs are sent to the stdout. [Default: Info]' - type: string - prometheusMetricsPort: - description: 'PrometheusMetricsPort is the TCP port that the Prometheus - metrics server should bind to. Set to 0 to disable. [Default: - 9094]' - type: integer - required: - - controllers - type: object - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: networkpolicies.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: NetworkPolicy - listKind: NetworkPolicyList - plural: networkpolicies - singular: networkpolicy - preserveUnknownFields: false - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - egress: - description: The ordered set of egress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with \"Not\". All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, global() - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a exact or a prefix match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, global() - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - required: - - action - type: object - type: array - ingress: - description: The ordered set of ingress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with \"Not\". All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, global() - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a exact or a prefix match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, global() - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - required: - - action - type: object - type: array - order: - description: Order is an optional field that specifies the order in - which the policy is applied. Policies with higher "order" are applied - after those with lower order. If the order is omitted, it may be - considered to be "infinite" - i.e. the policy will be applied last. Policies - with identical order will be applied in alphanumerical order based - on the Policy "Name". - type: number - selector: - description: "The selector is an expression used to pick pick out - the endpoints that the policy should be applied to. \n Selector - expressions follow this syntax: \n \tlabel == \"string_literal\" - \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" - \ -> not equal; also matches if label is not present \tlabel in - { \"a\", \"b\", \"c\", ... } -> true if the value of label X is - one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", - ... } -> true if the value of label X is not one of \"a\", \"b\", - \"c\" \thas(label_name) -> True if that label is present \t! expr - -> negation of expr \texpr && expr -> Short-circuit and \texpr - || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() - or the empty selector -> matches all endpoints. \n Label names are - allowed to contain alphanumerics, -, _ and /. String literals are - more permissive but they do not support escape characters. \n Examples - (with made-up labels): \n \ttype == \"webserver\" && deployment - == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != - \"dev\" \t! has(label_name)" - type: string - serviceAccountSelector: - description: ServiceAccountSelector is an optional field for an expression - used to select a pod based on service accounts. - type: string - types: - description: "Types indicates whether this policy applies to ingress, - or to egress, or to both. When not explicitly specified (and so - the value on creation is empty or nil), Calico defaults Types according - to what Ingress and Egress are present in the policy. The default - is: \n - [ PolicyTypeIngress ], if there are no Egress rules (including - the case where there are also no Ingress rules) \n - [ PolicyTypeEgress - ], if there are Egress rules but no Ingress rules \n - [ PolicyTypeIngress, - PolicyTypeEgress ], if there are both Ingress and Egress rules. - \n When the policy is read back again, Types will always be one - of these values, never empty or nil." - items: - description: PolicyType enumerates the possible values of the PolicySpec - Types field. - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: networksets.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: NetworkSet - listKind: NetworkSetList - plural: networksets - singular: networkset - preserveUnknownFields: false - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: NetworkSetSpec contains the specification for a NetworkSet - resource. - properties: - nets: - description: The list of IP networks that belong to this set. - items: - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/calico-kube-controllers-rbac.yaml -# Include a clusterrole for the kube-controllers component, -# and bind it to the calico-kube-controllers serviceaccount. -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-kube-controllers -rules: - # Nodes are watched to monitor for deletions. - - apiGroups: [""] - resources: - - nodes - verbs: - - watch - - list - - get - # Pods are watched to check for existence as part of IPAM controller. - - apiGroups: [""] - resources: - - pods - verbs: - - get - - list - - watch - # IPAM resources are manipulated in response to node and block updates, as well as periodic triggers. - - apiGroups: ["crd.projectcalico.org"] - resources: - - ipreservations - verbs: - - list - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - - ipamblocks - - ipamhandles - verbs: - - get - - list - - create - - update - - delete - - watch - # Pools are watched to maintain a mapping of blocks to IP pools. - - apiGroups: ["crd.projectcalico.org"] - resources: - - ippools - verbs: - - list - - watch - # kube-controllers manages hostendpoints. - - apiGroups: ["crd.projectcalico.org"] - resources: - - hostendpoints - verbs: - - get - - list - - create - - update - - delete - # Needs access to update clusterinformations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - clusterinformations - verbs: - - get - - list - - create - - update - - watch - # KubeControllersConfiguration is where it gets its config - - apiGroups: ["crd.projectcalico.org"] - resources: - - kubecontrollersconfigurations - verbs: - # read its own config - - get - # create a default if none exists - - create - # update status - - update - # watch for changes - - watch ---- -# Source: calico/templates/calico-node-rbac.yaml -# Include a clusterrole for the calico-node DaemonSet, -# and bind it to the calico-node serviceaccount. -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-node -rules: - # Used for creating service account tokens to be used by the CNI plugin - - apiGroups: [""] - resources: - - serviceaccounts/token - resourceNames: - - calico-cni-plugin - verbs: - - create - # The CNI plugin needs to get pods, nodes, and namespaces. - - apiGroups: [""] - resources: - - pods - - nodes - - namespaces - verbs: - - get - # EndpointSlices are used for Service-based network policy rule - # enforcement. - - apiGroups: ["discovery.k8s.io"] - resources: - - endpointslices - verbs: - - watch - - list - - apiGroups: [""] - resources: - - endpoints - - services - verbs: - # Used to discover service IPs for advertisement. - - watch - - list - # Used to discover Typhas. - - get - # Pod CIDR auto-detection on kubeadm needs access to config maps. - - apiGroups: [""] - resources: - - configmaps - verbs: - - get - - apiGroups: [""] - resources: - - nodes/status - verbs: - # Needed for clearing NodeNetworkUnavailable flag. - - patch - # Calico stores some configuration information in node annotations. - - update - # Watch for changes to Kubernetes NetworkPolicies. - - apiGroups: ["networking.k8s.io"] - resources: - - networkpolicies - verbs: - - watch - - list - # Used by Calico for policy information. - - apiGroups: [""] - resources: - - pods - - namespaces - - serviceaccounts - verbs: - - list - - watch - # The CNI plugin patches pods/status. - - apiGroups: [""] - resources: - - pods/status - verbs: - - patch - # Calico monitors various CRDs for config. - - apiGroups: ["crd.projectcalico.org"] - resources: - - globalfelixconfigs - - felixconfigurations - - bgppeers - - bgpfilters - - globalbgpconfigs - - bgpconfigurations - - ippools - - ipreservations - - ipamblocks - - globalnetworkpolicies - - globalnetworksets - - networkpolicies - - networksets - - clusterinformations - - hostendpoints - - blockaffinities - - caliconodestatuses - verbs: - - get - - list - - watch - # Calico must create and update some CRDs on startup. - - apiGroups: ["crd.projectcalico.org"] - resources: - - ippools - - felixconfigurations - - clusterinformations - verbs: - - create - - update - # Calico must update some CRDs. - - apiGroups: [ "crd.projectcalico.org" ] - resources: - - caliconodestatuses - verbs: - - update - # Calico stores some configuration information on the node. - - apiGroups: [""] - resources: - - nodes - verbs: - - get - - list - - watch - # These permissions are only required for upgrade from v2.6, and can - # be removed after upgrade or on fresh installations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - bgpconfigurations - - bgppeers - verbs: - - create - - update - # These permissions are required for Calico CNI to perform IPAM allocations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - - ipamblocks - - ipamhandles - verbs: - - get - - list - - create - - update - - delete - # The CNI plugin and calico/node need to be able to create a default - # IPAMConfiguration - - apiGroups: ["crd.projectcalico.org"] - resources: - - ipamconfigs - verbs: - - get - - create - # Block affinities must also be watchable by confd for route aggregation. - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - verbs: - - watch - # The Calico IPAM migration needs to get daemonsets. These permissions can be - # removed if not upgrading from an installation using host-local IPAM. - - apiGroups: ["apps"] - resources: - - daemonsets - verbs: - - get ---- -# Source: calico/templates/calico-node-rbac.yaml -# CNI cluster role -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-cni-plugin -rules: - - apiGroups: [""] - resources: - - pods - - nodes - - namespaces - verbs: - - get - - apiGroups: [""] - resources: - - pods/status - verbs: - - patch - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - - ipamblocks - - ipamhandles - - clusterinformations - - ippools - - ipreservations - - ipamconfigs - verbs: - - get - - list - - create - - update - - delete ---- -# Source: calico/templates/calico-kube-controllers-rbac.yaml -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-kube-controllers -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-kube-controllers -subjects: -- kind: ServiceAccount - name: calico-kube-controllers - namespace: kube-system ---- -# Source: calico/templates/calico-node-rbac.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: calico-node -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-node -subjects: -- kind: ServiceAccount - name: calico-node - namespace: kube-system ---- -# Source: calico/templates/calico-node-rbac.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: calico-cni-plugin -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-cni-plugin -subjects: -- kind: ServiceAccount - name: calico-cni-plugin - namespace: kube-system - -{{ if .TyphaEnabled }} ---- -# Source: calico/templates/calico-typha.yaml -# This manifest creates a Service, which will be backed by Calico's Typha daemon. -# Typha sits in between Felix and the API server, reducing Calico's load on the API server. - -apiVersion: v1 -kind: Service -metadata: - name: calico-typha - namespace: kube-system - labels: - k8s-app: calico-typha -spec: - ports: - - port: 5473 - protocol: TCP - targetPort: calico-typha - name: calico-typha - selector: - k8s-app: calico-typha - -{{ end }} ---- -# Source: calico/templates/calico-node.yaml -# This manifest installs the calico-node container, as well -# as the CNI plugins and network config on -# each master and worker node in a Kubernetes cluster. -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: calico-node - namespace: kube-system - labels: - k8s-app: calico-node -spec: - selector: - matchLabels: - k8s-app: calico-node - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - template: - metadata: - labels: - k8s-app: calico-node - spec: - nodeSelector: - kubernetes.io/os: linux - hostNetwork: true - tolerations: - # Make sure calico-node gets scheduled on all nodes. - - effect: NoSchedule - operator: Exists - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists - serviceAccountName: calico-node - # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force - # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. - terminationGracePeriodSeconds: 0 - priorityClassName: system-node-critical - initContainers: - # This container performs upgrade from host-local IPAM to calico-ipam. - # It can be deleted if this is a fresh installation, or if you have already - # upgraded to use calico-ipam. - - name: upgrade-ipam - image: {{ .CalicoCniImage }} - imagePullPolicy: IfNotPresent - command: ["/opt/cni/bin/calico-ipam", "-upgrade"] - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CALICO_NETWORKING_BACKEND - valueFrom: - configMapKeyRef: - name: calico-config - key: calico_backend - volumeMounts: - - mountPath: /var/lib/cni/networks - name: host-local-net-dir - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - securityContext: - privileged: true - # This container installs the CNI binaries - # and CNI network config file on each node. - - name: install-cni - image: {{ .CalicoCniImage }} - imagePullPolicy: IfNotPresent - command: ["/opt/cni/bin/install"] - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - # Name of the CNI config file to create. - - name: CNI_CONF_NAME - value: "10-calico.conflist" - # The CNI network config to install on each node. - - name: CNI_NETWORK_CONFIG - valueFrom: - configMapKeyRef: - name: calico-config - key: cni_network_config - # Set the hostname based on the k8s node name. - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # CNI MTU Config variable - - name: CNI_MTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Prevents the container from sleeping forever. - - name: SLEEP - value: "false" - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - - mountPath: /host/etc/cni/net.d - name: cni-net-dir - securityContext: - privileged: true - # This init container mounts the necessary filesystems needed by the BPF data plane - # i.e. bpf at /sys/fs/bpf and cgroup2 at /run/calico/cgroup. Calico-node initialisation is executed - # in best effort fashion, i.e. no failure for errors, to not disrupt pod creation in iptable mode. - - name: "mount-bpffs" - image: {{ .CalicoNodeImage }} - imagePullPolicy: IfNotPresent - command: ["calico-node", "-init", "-best-effort"] - volumeMounts: - - mountPath: /sys/fs - name: sys-fs - # Bidirectional is required to ensure that the new mount we make at /sys/fs/bpf propagates to the host - # so that it outlives the init container. - mountPropagation: Bidirectional - - mountPath: /var/run/calico - name: var-run-calico - # Bidirectional is required to ensure that the new mount we make at /run/calico/cgroup propagates to the host - # so that it outlives the init container. - mountPropagation: Bidirectional - # Mount /proc/ from host which usually is an init program at /nodeproc. It's needed by mountns binary, - # executed by calico-node, to mount root cgroup2 fs at /run/calico/cgroup to attach CTLB programs correctly. - - mountPath: /nodeproc - name: nodeproc - readOnly: true - securityContext: - privileged: true - containers: - # Runs calico-node container on each Kubernetes node. This - # container programs network policy and routes on each - # host. - - name: calico-node - image: {{ .CalicoNodeImage }} - imagePullPolicy: IfNotPresent - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - # Use Kubernetes API as the backing datastore. - - name: DATASTORE_TYPE - value: "kubernetes" -{{ if .TyphaEnabled }} - # Typha support: controlled by the ConfigMap. - - name: FELIX_TYPHAK8SSERVICENAME - valueFrom: - configMapKeyRef: - name: calico-config - key: typha_service_name -{{ end }} - # Wait for the datastore. - - name: WAIT_FOR_DATASTORE - value: "true" - # Set based on the k8s node name. - - name: NODENAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # Choose the backend to use. - - name: CALICO_NETWORKING_BACKEND - valueFrom: - configMapKeyRef: - name: calico-config - key: calico_backend - # Cluster type to identify the deployment type - - name: CLUSTER_TYPE - value: "k8s,bgp" - # Auto-detect the BGP IP address. - - name: NODEIP - valueFrom: - fieldRef: - fieldPath: status.hostIP - - name: IP_AUTODETECTION_METHOD - value: "can-reach=$(NODEIP)" - - name: IP - value: "autodetect" - # Enable IPIP - - name: CALICO_IPV4POOL_IPIP - value: "{{ .IPIPMode }}" - # Enable or Disable VXLAN on the default IP pool. - - name: CALICO_IPV4POOL_VXLAN - value: "{{ .VXLANMode }}" -{{- if .IPV4POOLNATOUTGOING }} - - name: CALICO_IPV4POOL_NAT_OUTGOING - value: "true" -{{- else }} - - name: CALICO_IPV4POOL_NAT_OUTGOING - value: "false" -{{- end }} - # Enable or Disable VXLAN on the default IPv6 IP pool. - - name: CALICO_IPV6POOL_VXLAN - value: "Never" - # Set MTU for tunnel device used if ipip is enabled - - name: FELIX_IPINIPMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Set MTU for the VXLAN tunnel device. - - name: FELIX_VXLANMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Set MTU for the Wireguard tunnel device. - - name: FELIX_WIREGUARDMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu -{{- if .DefaultIPPOOL }} - # The default IPv4 pool to create on startup if none exists. Pod IPs will be - # chosen from this range. Changing this value after installation will have - # no effect. - - name: CALICO_IPV4POOL_CIDR - value: "{{ .KubePodsCIDR }}" - - name: CALICO_IPV4POOL_BLOCK_SIZE - value: "{{ .NodeCidrMaskSize }}" -{{- else }} - - name: NO_DEFAULT_POOLS - value: "true" - - name: CALICO_IPV4POOL_CIDR - value: "" - - name: CALICO_IPV6POOL_CIDR - value: "" -{{- end }} - - name: CALICO_DISABLE_FILE_LOGGING - value: "true" - # Set Felix endpoint to host default action to ACCEPT. - - name: FELIX_DEFAULTENDPOINTTOHOSTACTION - value: "ACCEPT" - # Disable IPv6 on Kubernetes. - - name: FELIX_IPV6SUPPORT - value: "false" - - name: FELIX_HEALTHENABLED - value: "true" - - name: FELIX_DEVICEROUTESOURCEADDRESS - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.hostIP - securityContext: - privileged: true - resources: - requests: - cpu: 250m - lifecycle: - preStop: - exec: - command: - - /bin/calico-node - - -shutdown - livenessProbe: - exec: - command: - - /bin/calico-node - - -felix-live - - -bird-live - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 - timeoutSeconds: 10 - readinessProbe: - exec: - command: - - /bin/calico-node - - -felix-ready - - -bird-ready - periodSeconds: 10 - timeoutSeconds: 10 - volumeMounts: - # For maintaining CNI plugin API credentials. - - mountPath: /host/etc/cni/net.d - name: cni-net-dir - readOnly: false - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - mountPath: /run/xtables.lock - name: xtables-lock - readOnly: false - - mountPath: /var/run/calico - name: var-run-calico - readOnly: false - - mountPath: /var/lib/calico - name: var-lib-calico - readOnly: false - - name: policysync - mountPath: /var/run/nodeagent - # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the - # parent directory. - - name: bpffs - mountPath: /sys/fs/bpf - - name: cni-log-dir - mountPath: /var/log/calico/cni - readOnly: true - volumes: - # Used by calico-node. - - name: lib-modules - hostPath: - path: /lib/modules - - name: var-run-calico - hostPath: - path: /var/run/calico - - name: var-lib-calico - hostPath: - path: /var/lib/calico - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: FileOrCreate - - name: sys-fs - hostPath: - path: /sys/fs/ - type: DirectoryOrCreate - - name: bpffs - hostPath: - path: /sys/fs/bpf - type: Directory - # mount /proc at /nodeproc to be used by mount-bpffs initContainer to mount root cgroup2 fs. - - name: nodeproc - hostPath: - path: /proc - # Used to install CNI. - - name: cni-bin-dir - hostPath: - path: /opt/cni/bin - - name: cni-net-dir - hostPath: - path: /etc/cni/net.d - # Used to access CNI logs. - - name: cni-log-dir - hostPath: - path: /var/log/calico/cni - # Mount in the directory for host-local IPAM allocations. This is - # used when upgrading from host-local to calico-ipam, and can be removed - # if not using the upgrade-ipam init container. - - name: host-local-net-dir - hostPath: - path: /var/lib/cni/networks - # Used to create per-pod Unix Domain Sockets - - name: policysync - hostPath: - type: DirectoryOrCreate - path: /var/run/nodeagent ---- -# Source: calico/templates/calico-kube-controllers.yaml -# See https://github.com/projectcalico/kube-controllers -apiVersion: apps/v1 -kind: Deployment -metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers -spec: - # The controllers can only have a single active instance. - replicas: 1 - selector: - matchLabels: - k8s-app: calico-kube-controllers - strategy: - type: Recreate - template: - metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers - spec: - nodeSelector: - kubernetes.io/os: linux - tolerations: - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - key: node-role.kubernetes.io/master - effect: NoSchedule - - key: node-role.kubernetes.io/control-plane - effect: NoSchedule - serviceAccountName: calico-kube-controllers - priorityClassName: system-cluster-critical - containers: - - name: calico-kube-controllers - image: {{ .CalicoControllersImage }} - imagePullPolicy: IfNotPresent - env: - # Choose which controllers to run. - - name: ENABLED_CONTROLLERS - value: node - - name: DATASTORE_TYPE - value: kubernetes - livenessProbe: - exec: - command: - - /usr/bin/check-status - - -l - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 - timeoutSeconds: 10 - readinessProbe: - exec: - command: - - /usr/bin/check-status - - -r - periodSeconds: 10 - -{{ if .TyphaEnabled }} ---- -# Source: calico/templates/calico-typha.yaml -# This manifest creates a Deployment of Typha to back the above service. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: calico-typha - namespace: kube-system - labels: - k8s-app: calico-typha -spec: - # Number of Typha replicas. To enable Typha, set this to a non-zero value *and* set the - # typha_service_name variable in the calico-config ConfigMap above. - # - # We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential - # (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In - # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade. - replicas: 1 - revisionHistoryLimit: 2 - selector: - matchLabels: - k8s-app: calico-typha - strategy: - rollingUpdate: - # 100% surge allows a complete up-level set of typha instances to start and become ready, - # which in turn allows all the back-level typha instances to start shutting down. This - # means that connections tend to bounce directly from a back-level instance to an up-level - # instance. - maxSurge: 100% - # In case the cluster is unable to schedule extra surge instances, allow at most one instance - # to shut down to make room. You can set this to 0 if you're sure there'll always be enough room to - # schedule extra typha instances during an upgrade (because setting it to 0 blocks shutdown until - # up-level typha instances are online and ready). - maxUnavailable: 1 - type: RollingUpdate - template: - metadata: - labels: - k8s-app: calico-typha - annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict: 'true' - spec: - nodeSelector: - kubernetes.io/os: linux - hostNetwork: true - # Typha supports graceful shut down, disconnecting clients slowly during the grace period. - # The TYPHA_SHUTDOWNTIMEOUTSECS env var should be kept in sync with this value. - terminationGracePeriodSeconds: 300 - tolerations: - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - # Since Calico can't network a pod until Typha is up, we need to run Typha itself - # as a host-networked pod. - serviceAccountName: calico-node - priorityClassName: system-cluster-critical - # fsGroup allows using projected serviceaccount tokens as described here kubernetes/kubernetes#82573 - securityContext: - fsGroup: 65534 - containers: - - image: {{ .CalicoTyphaImage }} - imagePullPolicy: IfNotPresent - name: calico-typha - ports: - - containerPort: 5473 - name: calico-typha - protocol: TCP - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - # Enable "info" logging by default. Can be set to "debug" to increase verbosity. - - name: TYPHA_LOGSEVERITYSCREEN - value: "info" - # Disable logging to file and syslog since those don't make sense in Kubernetes. - - name: TYPHA_LOGFILEPATH - value: "none" - - name: TYPHA_LOGSEVERITYSYS - value: "none" - # Monitor the Kubernetes API to find the number of running instances and rebalance - # connections. - - name: TYPHA_CONNECTIONREBALANCINGMODE - value: "kubernetes" - - name: TYPHA_DATASTORETYPE - value: "kubernetes" - - name: TYPHA_HEALTHENABLED - value: "true" - # Set this to the same value as terminationGracePeriodSeconds; it tells Typha how much time - # it has to shut down. - - name: TYPHA_SHUTDOWNTIMEOUTSECS - value: "300" - # Uncomment these lines to enable prometheus metrics. Since Typha is host-networked, - # this opens a port on the host, which may need to be secured. - #- name: TYPHA_PROMETHEUSMETRICSENABLED - # value: "true" - #- name: TYPHA_PROMETHEUSMETRICSPORT - # value: "9093" - livenessProbe: - httpGet: - path: /liveness - port: 9098 - host: localhost - periodSeconds: 30 - initialDelaySeconds: 30 - timeoutSeconds: 10 - securityContext: - runAsNonRoot: true - allowPrivilegeEscalation: false - readinessProbe: - httpGet: - path: /readiness - port: 9098 - host: localhost - periodSeconds: 10 - timeoutSeconds: 10 - -{{ end }} - - `))) diff --git a/cmd/kk/pkg/plugins/network/templates/calico_v1.16-.go b/cmd/kk/pkg/plugins/network/templates/calico_v1.16-.go deleted file mode 100644 index dc56d7c67..000000000 --- a/cmd/kk/pkg/plugins/network/templates/calico_v1.16-.go +++ /dev/null @@ -1,1037 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var CalicoOld = template.Must(template.New("network-plugin.yaml").Parse( - dedent.Dedent(`--- -# Source: calico/templates/calico-config.yaml -# This ConfigMap is used to configure a self-hosted Calico installation. -kind: ConfigMap -apiVersion: v1 -metadata: - name: calico-config - namespace: kube-system -data: - # You must set a non-zero value for Typha replicas below. - typha_service_name: {{ if .TyphaEnabled }}"calico-typha"{{ else }}"none"{{ end }} - # Configure the backend to use. - calico_backend: "bird" - # Configure the MTU to use for workload interfaces and the - # tunnels. For IPIP, set to your network MTU - 20; for VXLAN - # set to your network MTU - 50. - veth_mtu: "{{ .VethMTU }}" - - # The CNI network configuration to install on each node. The special - # values in this config will be automatically populated. - cni_network_config: |- - { - "name": "k8s-pod-network", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "calico", - "log_level": "info", - "datastore_type": "kubernetes", - "nodename": "__KUBERNETES_NODE_NAME__", - "mtu": __CNI_MTU__, - "ipam": { - "type": "calico-ipam" - }, - "policy": { - "type": "k8s" - }, - "kubernetes": { - "kubeconfig": "__KUBECONFIG_FILEPATH__" - } - }, - { - "type": "portmap", - "snat": true, - "capabilities": {"portMappings": true} - }, - { - "type": "bandwidth", - "capabilities": {"bandwidth": true} - } - ] - } - ---- -# Source: calico/templates/kdd-crds.yaml - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: bgpconfigurations.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: BGPConfiguration - plural: bgpconfigurations - singular: bgpconfiguration - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: bgppeers.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: BGPPeer - plural: bgppeers - singular: bgppeer - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: blockaffinities.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: BlockAffinity - plural: blockaffinities - singular: blockaffinity - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: clusterinformations.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: ClusterInformation - plural: clusterinformations - singular: clusterinformation - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: felixconfigurations.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: FelixConfiguration - plural: felixconfigurations - singular: felixconfiguration - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: globalnetworkpolicies.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: GlobalNetworkPolicy - plural: globalnetworkpolicies - singular: globalnetworkpolicy - shortNames: - - gnp - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: globalnetworksets.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: GlobalNetworkSet - plural: globalnetworksets - singular: globalnetworkset - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: hostendpoints.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: HostEndpoint - plural: hostendpoints - singular: hostendpoint - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: ipamblocks.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: IPAMBlock - plural: ipamblocks - singular: ipamblock - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: ipamconfigs.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: IPAMConfig - plural: ipamconfigs - singular: ipamconfig - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: ipamhandles.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: IPAMHandle - plural: ipamhandles - singular: ipamhandle - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: ippools.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: IPPool - plural: ippools - singular: ippool - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: kubecontrollersconfigurations.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: KubeControllersConfiguration - plural: kubecontrollersconfigurations - singular: kubecontrollersconfiguration ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: networkpolicies.crd.projectcalico.org -spec: - scope: Namespaced - group: crd.projectcalico.org - version: v1 - names: - kind: NetworkPolicy - plural: networkpolicies - singular: networkpolicy - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: networksets.crd.projectcalico.org -spec: - scope: Namespaced - group: crd.projectcalico.org - version: v1 - names: - kind: NetworkSet - plural: networksets - singular: networkset - ---- ---- -# Source: calico/templates/rbac.yaml - -# Include a clusterrole for the kube-controllers component, -# and bind it to the calico-kube-controllers serviceaccount. -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-kube-controllers -rules: - # Nodes are watched to monitor for deletions. - - apiGroups: [""] - resources: - - nodes - verbs: - - watch - - list - - get - # Pods are queried to check for existence. - - apiGroups: [""] - resources: - - pods - verbs: - - get - # IPAM resources are manipulated when nodes are deleted. - - apiGroups: ["crd.projectcalico.org"] - resources: - - ippools - verbs: - - list - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - - ipamblocks - - ipamhandles - verbs: - - get - - list - - create - - update - - delete - # kube-controllers manages hostendpoints. - - apiGroups: ["crd.projectcalico.org"] - resources: - - hostendpoints - verbs: - - get - - list - - create - - update - - delete - # Needs access to update clusterinformations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - clusterinformations - verbs: - - get - - create - - update - # KubeControllersConfiguration is where it gets its config - - apiGroups: ["crd.projectcalico.org"] - resources: - - kubecontrollersconfigurations - verbs: - # read its own config - - get - # create a default if none exists - - create - # update status - - update - # watch for changes - - watch ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-kube-controllers -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-kube-controllers -subjects: -- kind: ServiceAccount - name: calico-kube-controllers - namespace: kube-system ---- -# Include a clusterrole for the calico-node DaemonSet, -# and bind it to the calico-node serviceaccount. -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-node -rules: - # The CNI plugin needs to get pods, nodes, and namespaces. - - apiGroups: [""] - resources: - - pods - - nodes - - namespaces - verbs: - - get - - apiGroups: [""] - resources: - - endpoints - - services - verbs: - # Used to discover service IPs for advertisement. - - watch - - list - # Used to discover Typhas. - - get - # Pod CIDR auto-detection on kubeadm needs access to config maps. - - apiGroups: [""] - resources: - - configmaps - verbs: - - get - - apiGroups: [""] - resources: - - nodes/status - verbs: - # Needed for clearing NodeNetworkUnavailable flag. - - patch - # Calico stores some configuration information in node annotations. - - update - # Watch for changes to Kubernetes NetworkPolicies. - - apiGroups: ["networking.k8s.io"] - resources: - - networkpolicies - verbs: - - watch - - list - # Used by Calico for policy information. - - apiGroups: [""] - resources: - - pods - - namespaces - - serviceaccounts - verbs: - - list - - watch - # The CNI plugin patches pods/status. - - apiGroups: [""] - resources: - - pods/status - verbs: - - patch - # Calico monitors various CRDs for config. - - apiGroups: ["crd.projectcalico.org"] - resources: - - globalfelixconfigs - - felixconfigurations - - bgppeers - - globalbgpconfigs - - bgpconfigurations - - ippools - - ipamblocks - - globalnetworkpolicies - - globalnetworksets - - networkpolicies - - networksets - - clusterinformations - - hostendpoints - - blockaffinities - verbs: - - get - - list - - watch - # Calico must create and update some CRDs on startup. - - apiGroups: ["crd.projectcalico.org"] - resources: - - ippools - - felixconfigurations - - clusterinformations - verbs: - - create - - update - # Calico stores some configuration information on the node. - - apiGroups: [""] - resources: - - nodes - verbs: - - get - - list - - watch - # These permissions are only requried for upgrade from v2.6, and can - # be removed after upgrade or on fresh installations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - bgpconfigurations - - bgppeers - verbs: - - create - - update - # These permissions are required for Calico CNI to perform IPAM allocations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - - ipamblocks - - ipamhandles - verbs: - - get - - list - - create - - update - - delete - - apiGroups: ["crd.projectcalico.org"] - resources: - - ipamconfigs - verbs: - - get - # Block affinities must also be watchable by confd for route aggregation. - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - verbs: - - watch - # The Calico IPAM migration needs to get daemonsets. These permissions can be - # removed if not upgrading from an installation using host-local IPAM. - - apiGroups: ["apps"] - resources: - - daemonsets - verbs: - - get - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: calico-node -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-node -subjects: -- kind: ServiceAccount - name: calico-node - namespace: kube-system - -{{ if .TyphaEnabled }} ---- -# Source: calico/templates/calico-typha.yaml -# This manifest creates a Service, which will be backed by Calico's Typha daemon. -# Typha sits in between Felix and the API server, reducing Calico's load on the API server. - -apiVersion: v1 -kind: Service -metadata: - name: calico-typha - namespace: kube-system - labels: - k8s-app: calico-typha -spec: - ports: - - port: 5473 - protocol: TCP - targetPort: calico-typha - name: calico-typha - selector: - k8s-app: calico-typha - ---- - -# This manifest creates a Deployment of Typha to back the above service. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: calico-typha - namespace: kube-system - labels: - k8s-app: calico-typha -spec: - # Number of Typha replicas. To enable Typha, set this to a non-zero value *and* set the - # typha_service_name variable in the calico-config ConfigMap above. - # - # We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential - # (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In - # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade. - replicas: 1 - revisionHistoryLimit: 2 - selector: - matchLabels: - k8s-app: calico-typha - template: - metadata: - labels: - k8s-app: calico-typha - annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict: 'true' - spec: - nodeSelector: - kubernetes.io/os: linux - hostNetwork: true - tolerations: - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - # Since Calico can't network a pod until Typha is up, we need to run Typha itself - # as a host-networked pod. - serviceAccountName: calico-node - priorityClassName: system-cluster-critical - # fsGroup allows using projected serviceaccount tokens as described here kubernetes/kubernetes#82573 - securityContext: - fsGroup: 65534 - containers: - - image: {{ .CalicoTyphaImage }} - name: calico-typha - ports: - - containerPort: 5473 - name: calico-typha - protocol: TCP - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - # Enable "info" logging by default. Can be set to "debug" to increase verbosity. - - name: TYPHA_LOGSEVERITYSCREEN - value: "info" - # Disable logging to file and syslog since those don't make sense in Kubernetes. - - name: TYPHA_LOGFILEPATH - value: "none" - - name: TYPHA_LOGSEVERITYSYS - value: "none" - # Monitor the Kubernetes API to find the number of running instances and rebalance - # connections. - - name: TYPHA_CONNECTIONREBALANCINGMODE - value: "kubernetes" - - name: TYPHA_DATASTORETYPE - value: "kubernetes" - - name: TYPHA_HEALTHENABLED - value: "true" - # Uncomment these lines to enable prometheus metrics. Since Typha is host-networked, - # this opens a port on the host, which may need to be secured. - #- name: TYPHA_PROMETHEUSMETRICSENABLED - # value: "true" - #- name: TYPHA_PROMETHEUSMETRICSPORT - # value: "9093" - livenessProbe: - httpGet: - path: /liveness - port: 9098 - host: localhost - periodSeconds: 30 - initialDelaySeconds: 30 - securityContext: - runAsNonRoot: true - allowPrivilegeEscalation: false - readinessProbe: - httpGet: - path: /readiness - port: 9098 - host: localhost - periodSeconds: 10 - - --- - - # This manifest creates a Pod Disruption Budget for Typha to allow K8s Cluster Autoscaler to evict - - apiVersion: policy/v1beta1 - kind: PodDisruptionBudget - metadata: - name: calico-typha - namespace: kube-system - labels: - k8s-app: calico-typha - spec: - maxUnavailable: 1 - selector: - matchLabels: - k8s-app: calico-typha - {{ end }} - ---- -# Source: calico/templates/calico-node.yaml -# This manifest installs the calico-node container, as well -# as the CNI plugins and network config on -# each master and worker node in a Kubernetes cluster. -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: calico-node - namespace: kube-system - labels: - k8s-app: calico-node -spec: - selector: - matchLabels: - k8s-app: calico-node - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - template: - metadata: - labels: - k8s-app: calico-node - spec: - nodeSelector: - kubernetes.io/os: linux - hostNetwork: true - tolerations: - # Make sure calico-node gets scheduled on all nodes. - - effect: NoSchedule - operator: Exists - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists - serviceAccountName: calico-node - # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force - # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. - terminationGracePeriodSeconds: 0 - priorityClassName: system-node-critical - initContainers: - # This container performs upgrade from host-local IPAM to calico-ipam. - # It can be deleted if this is a fresh installation, or if you have already - # upgraded to use calico-ipam. - - name: upgrade-ipam - image: {{ .CalicoCniImage }} - command: ["/opt/cni/bin/calico-ipam", "-upgrade"] - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CALICO_NETWORKING_BACKEND - valueFrom: - configMapKeyRef: - name: calico-config - key: calico_backend - volumeMounts: - - mountPath: /var/lib/cni/networks - name: host-local-net-dir - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - securityContext: - privileged: true - # This container installs the CNI binaries - # and CNI network config file on each node. - - name: install-cni - image: {{ .CalicoCniImage }} - command: ["/opt/cni/bin/install"] - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - # Name of the CNI config file to create. - - name: CNI_CONF_NAME - value: "10-calico.conflist" - # The CNI network config to install on each node. - - name: CNI_NETWORK_CONFIG - valueFrom: - configMapKeyRef: - name: calico-config - key: cni_network_config - # Set the hostname based on the k8s node name. - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # CNI MTU Config variable - - name: CNI_MTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Prevents the container from sleeping forever. - - name: SLEEP - value: "false" - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - - mountPath: /host/etc/cni/net.d - name: cni-net-dir - securityContext: - privileged: true - # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes - # to communicate with Felix over the Policy Sync API. - - name: flexvol-driver - image: {{ .CalicoFlexvolImage }} - volumeMounts: - - name: flexvol-driver-host - mountPath: /host/driver - securityContext: - privileged: true - containers: - # Runs calico-node container on each Kubernetes node. This - # container programs network policy and routes on each - # host. - - name: calico-node - image: {{ .CalicoNodeImage }} - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - # Use Kubernetes API as the backing datastore. - - name: DATASTORE_TYPE - value: "kubernetes" -{{ if .TyphaEnabled }} - # Typha support: controlled by the ConfigMap. - - name: FELIX_TYPHAK8SSERVICENAME - valueFrom: - configMapKeyRef: - name: calico-config - key: typha_service_name -{{ end }} - # Wait for the datastore. - - name: WAIT_FOR_DATASTORE - value: "true" - # Set based on the k8s node name. - - name: NODENAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # Choose the backend to use. - - name: CALICO_NETWORKING_BACKEND - valueFrom: - configMapKeyRef: - name: calico-config - key: calico_backend - # Cluster type to identify the deployment type - - name: CLUSTER_TYPE - value: "k8s,bgp" - # Auto-detect the BGP IP address. - - name: NODEIP - valueFrom: - fieldRef: - fieldPath: status.hostIP - - name: IP_AUTODETECTION_METHOD - value: "can-reach=$(NODEIP)" - - name: IP - value: "autodetect" - # Enable IPIP - - name: CALICO_IPV4POOL_IPIP - value: "{{ .IPIPMode }}" - # Enable or Disable VXLAN on the default IP pool. - - name: CALICO_IPV4POOL_VXLAN - value: "{{ .VXLANMode }}" - # Set MTU for tunnel device used if ipip is enabled - - name: FELIX_IPINIPMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Set MTU for the VXLAN tunnel device. - - name: FELIX_VXLANMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Set MTU for the Wireguard tunnel device. - - name: FELIX_WIREGUARDMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # The default IPv4 pool to create on startup if none exists. Pod IPs will be - # chosen from this range. Changing this value after installation will have - # no effect. - - name: CALICO_IPV4POOL_CIDR - value: "{{ .KubePodsCIDR }}" - - name: CALICO_IPV4POOL_BLOCK_SIZE - value: "{{ .NodeCidrMaskSize }}" - - name: CALICO_DISABLE_FILE_LOGGING - value: "true" - # Set Felix endpoint to host default action to ACCEPT. - - name: FELIX_DEFAULTENDPOINTTOHOSTACTION - value: "ACCEPT" - # Disable IPv6 on Kubernetes. - - name: FELIX_IPV6SUPPORT - value: "false" - # Set Felix logging to "info" - - name: FELIX_LOGSEVERITYSCREEN - value: "info" - - name: FELIX_HEALTHENABLED - value: "true" - - name: FELIX_DEVICEROUTESOURCEADDRESS - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.hostIP - securityContext: - privileged: true - resources: - requests: - cpu: 250m - livenessProbe: - exec: - command: - - /bin/calico-node - - -felix-live - - -bird-live - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 - readinessProbe: - exec: - command: - - /bin/calico-node - - -felix-ready - - -bird-ready - periodSeconds: 10 - volumeMounts: - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - mountPath: /run/xtables.lock - name: xtables-lock - readOnly: false - - mountPath: /var/run/calico - name: var-run-calico - readOnly: false - - mountPath: /var/lib/calico - name: var-lib-calico - readOnly: false - - name: policysync - mountPath: /var/run/nodeagent - # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the - # parent directory. - - name: sysfs - mountPath: /sys/fs/ - # Bidirectional means that, if we mount the BPF filesystem at /sys/fs/bpf it will propagate to the host. - # If the host is known to mount that filesystem already then Bidirectional can be omitted. - mountPropagation: Bidirectional - volumes: - # Used by calico-node. - - name: lib-modules - hostPath: - path: /lib/modules - - name: var-run-calico - hostPath: - path: /var/run/calico - - name: var-lib-calico - hostPath: - path: /var/lib/calico - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: FileOrCreate - - name: sysfs - hostPath: - path: /sys/fs/ - type: DirectoryOrCreate - # Used to install CNI. - - name: cni-bin-dir - hostPath: - path: /opt/cni/bin - - name: cni-net-dir - hostPath: - path: /etc/cni/net.d - # Mount in the directory for host-local IPAM allocations. This is - # used when upgrading from host-local to calico-ipam, and can be removed - # if not using the upgrade-ipam init container. - - name: host-local-net-dir - hostPath: - path: /var/lib/cni/networks - # Used to create per-pod Unix Domain Sockets - - name: policysync - hostPath: - type: DirectoryOrCreate - path: /var/run/nodeagent - # Used to install Flex Volume Driver - - name: flexvol-driver-host - hostPath: - type: DirectoryOrCreate - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds ---- - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-node - namespace: kube-system - ---- -# Source: calico/templates/calico-kube-controllers.yaml -# See https://github.com/projectcalico/kube-controllers -apiVersion: apps/v1 -kind: Deployment -metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers -spec: - # The controllers can only have a single active instance. - replicas: 1 - selector: - matchLabels: - k8s-app: calico-kube-controllers - strategy: - type: Recreate - template: - metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers - spec: - nodeSelector: - kubernetes.io/os: linux - tolerations: - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - key: node-role.kubernetes.io/master - effect: NoSchedule - serviceAccountName: calico-kube-controllers - priorityClassName: system-cluster-critical - containers: - - name: calico-kube-controllers - image: {{ .CalicoControllersImage }} - env: - # Choose which controllers to run. - - name: ENABLED_CONTROLLERS - value: node - - name: DATASTORE_TYPE - value: kubernetes - readinessProbe: - exec: - command: - - /usr/bin/check-status - - -r - ---- - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-kube-controllers - namespace: kube-system - - `))) diff --git a/cmd/kk/pkg/plugins/network/templates/cilium.go b/cmd/kk/pkg/plugins/network/templates/cilium.go deleted file mode 100644 index ba7f6870b..000000000 --- a/cmd/kk/pkg/plugins/network/templates/cilium.go +++ /dev/null @@ -1,688 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var Cilium = template.Must(template.New("network-plugin.yaml").Parse( - dedent.Dedent(`--- -# Source: cilium/charts/agent/templates/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: cilium - namespace: kube-system ---- -# Source: cilium/charts/operator/templates/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: cilium-operator - namespace: kube-system ---- -# Source: cilium/charts/config/templates/configmap.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: cilium-config - namespace: kube-system -data: - - # Identity allocation mode selects how identities are shared between cilium - # nodes by setting how they are stored. The options are "crd" or "kvstore". - # - "crd" stores identities in kubernetes as CRDs (custom resource definition). - # These can be queried with: - # kubectl get ciliumid - # - "kvstore" stores identities in a kvstore, etcd or consul, that is - # configured below. Cilium versions before 1.6 supported only the kvstore - # backend. Upgrades from these older cilium versions should continue using - # the kvstore by commenting out the identity-allocation-mode below, or - # setting it to "kvstore". - identity-allocation-mode: crd - - # If you want to run cilium in debug mode change this value to true - debug: "false" - - # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 - # address. - enable-ipv4: "true" - - # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 - # address. - enable-ipv6: "false" - enable-bpf-clock-probe: "true" - - # If you want cilium monitor to aggregate tracing for packets, set this level - # to "low", "medium", or "maximum". The higher the level, the less packets - # that will be seen in monitor output. - monitor-aggregation: medium - - # The monitor aggregation interval governs the typical time between monitor - # notification events for each allowed connection. - # - # Only effective when monitor aggregation is set to "medium" or higher. - monitor-aggregation-interval: 5s - - # The monitor aggregation flags determine which TCP flags which, upon the - # first observation, cause monitor notifications to be generated. - # - # Only effective when monitor aggregation is set to "medium" or higher. - monitor-aggregation-flags: all - # bpf-policy-map-max specified the maximum number of entries in endpoint - # policy map (per endpoint) - bpf-policy-map-max: "16384" - # Specifies the ratio (0.0-1.0) of total system memory to use for dynamic - # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. - bpf-map-dynamic-size-ratio: "0.0025" - - # Pre-allocation of map entries allows per-packet latency to be reduced, at - # the expense of up-front memory allocation for the entries in the maps. The - # default value below will minimize memory usage in the default installation; - # users who are sensitive to latency may consider setting this to "true". - # - # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore - # this option and behave as though it is set to "true". - # - # If this value is modified, then during the next Cilium startup the restore - # of existing endpoints and tracking of ongoing connections may be disrupted. - # This may lead to policy drops or a change in loadbalancing decisions for a - # connection for some time. Endpoints may need to be recreated to restore - # connectivity. - # - # If this option is set to "false" during an upgrade from 1.3 or earlier to - # 1.4 or later, then it may cause one-time disruptions during the upgrade. - preallocate-bpf-maps: "false" - - # Regular expression matching compatible Istio sidecar istio-proxy - # container image names - sidecar-istio-proxy-image: "cilium/istio_proxy" - - # Encapsulation mode for communication between nodes - # Possible values: - # - disabled - # - vxlan (default) - # - geneve - tunnel: vxlan - - # Name of the cluster. Only relevant when building a mesh of clusters. - cluster-name: default - - # wait-bpf-mount makes init container wait until bpf filesystem is mounted - wait-bpf-mount: "false" - - masquerade: "true" - enable-bpf-masquerade: "true" - enable-xt-socket-fallback: "true" - install-iptables-rules: "true" - auto-direct-node-routes: "false" - kube-proxy-replacement: "probe" - enable-health-check-nodeport: "true" - node-port-bind-protection: "true" - enable-auto-protect-node-port-range: "true" - enable-session-affinity: "true" - k8s-require-ipv4-pod-cidr: "true" - k8s-require-ipv6-pod-cidr: "false" - enable-endpoint-health-checking: "true" - enable-well-known-identities: "false" - enable-remote-node-identity: "true" - operator-api-serve-addr: "127.0.0.1:9234" - ipam: "cluster-pool" - cluster-pool-ipv4-cidr: "{{ .KubePodsCIDR }}" - cluster-pool-ipv4-mask-size: "{{ .NodeCidrMaskSize }}" - disable-cnp-status-updates: "true" ---- -# Source: cilium/charts/agent/templates/clusterrole.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cilium -rules: -- apiGroups: - - networking.k8s.io - resources: - - networkpolicies - verbs: - - get - - list - - watch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - namespaces - - services - - nodes - - endpoints - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - pods - - nodes - verbs: - - get - - list - - watch - - update -- apiGroups: - - "" - resources: - - nodes - - nodes/status - verbs: - - patch -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - create - - get - - list - - watch - - update -- apiGroups: - - cilium.io - resources: - - ciliumnetworkpolicies - - ciliumnetworkpolicies/status - - ciliumclusterwidenetworkpolicies - - ciliumclusterwidenetworkpolicies/status - - ciliumendpoints - - ciliumendpoints/status - - ciliumnodes - - ciliumnodes/status - - ciliumidentities -# deprecated remove in v1.9 - - ciliumidentities/status - verbs: - - '*' ---- -# Source: cilium/charts/operator/templates/clusterrole.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cilium-operator -rules: -- apiGroups: - - "" - resources: - # to automatically delete [core|kube]dns pods so that are starting to being - # managed by Cilium - - pods - verbs: - - get - - list - - watch - - delete -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - # to perform the translation of a CNP that contains ToGroup to its endpoints - - services - - endpoints - # to check apiserver connectivity - - namespaces - verbs: - - get - - list - - watch -- apiGroups: - - cilium.io - resources: - - ciliumnetworkpolicies - - ciliumnetworkpolicies/status - - ciliumclusterwidenetworkpolicies - - ciliumclusterwidenetworkpolicies/status - - ciliumendpoints - - ciliumendpoints/status - - ciliumnodes - - ciliumnodes/status - - ciliumidentities - - ciliumidentities/status - verbs: - - '*' -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - get - - list - - watch -# For cilium-operator running in HA mode. -# -# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election -# between mulitple running instances. -# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less -# common and fewer objects in the cluster watch "all Leases". -# The support for leases was introduced in coordination.k8s.io/v1 during Kubernetes 1.14 release. -# In Cilium we currently don't support HA mode for K8s version < 1.14. This condition make sure -# that we only authorize access to leases resources in supported K8s versions. -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - get - - update ---- -# Source: cilium/charts/agent/templates/clusterrolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: cilium -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cilium -subjects: -- kind: ServiceAccount - name: cilium - namespace: kube-system ---- -# Source: cilium/charts/operator/templates/clusterrolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: cilium-operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cilium-operator -subjects: -- kind: ServiceAccount - name: cilium-operator - namespace: kube-system ---- -# Source: cilium/charts/agent/templates/daemonset.yaml -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - k8s-app: cilium - name: cilium - namespace: kube-system -spec: - selector: - matchLabels: - k8s-app: cilium - template: - metadata: - annotations: - # This annotation plus the CriticalAddonsOnly toleration makes - # cilium to be a critical pod in the cluster, which ensures cilium - # gets priority scheduling. - # https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ - scheduler.alpha.kubernetes.io/critical-pod: "" - labels: - k8s-app: cilium - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - podAffinityTerm: - labelSelector: - matchExpressions: - - key: io.cilium/app - operator: In - values: - - operator - topologyKey: kubernetes.io/hostname - weight: 100 - containers: - - args: - - --config-dir=/tmp/cilium/config-map - command: - - cilium-agent - livenessProbe: - httpGet: - host: '127.0.0.1' - path: /healthz - port: 9876 - scheme: HTTP - httpHeaders: - - name: "brief" - value: "true" - failureThreshold: 10 - # The initial delay for the liveness probe is intentionally large to - # avoid an endless kill & restart cycle if in the event that the initial - # bootstrapping takes longer than expected. - initialDelaySeconds: 120 - periodSeconds: 30 - successThreshold: 1 - timeoutSeconds: 5 - readinessProbe: - httpGet: - host: '127.0.0.1' - path: /healthz - port: 9876 - scheme: HTTP - httpHeaders: - - name: "brief" - value: "true" - failureThreshold: 3 - initialDelaySeconds: 5 - periodSeconds: 30 - successThreshold: 1 - timeoutSeconds: 5 - env: - - name: K8S_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: CILIUM_K8S_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: CILIUM_FLANNEL_MASTER_DEVICE - valueFrom: - configMapKeyRef: - key: flannel-master-device - name: cilium-config - optional: true - - name: CILIUM_FLANNEL_UNINSTALL_ON_EXIT - valueFrom: - configMapKeyRef: - key: flannel-uninstall-on-exit - name: cilium-config - optional: true - - name: CILIUM_CLUSTERMESH_CONFIG - value: /var/lib/cilium/clustermesh/ - - name: CILIUM_CNI_CHAINING_MODE - valueFrom: - configMapKeyRef: - key: cni-chaining-mode - name: cilium-con - optional: true - - name: CILIUM_CUSTOM_CNI_CONF - valueFrom: - configMapKeyRef: - key: custom-cni-conf - name: cilium-config - optional: true - image: "{{ .CiliumImage }}" - imagePullPolicy: IfNotPresent - lifecycle: - postStart: - exec: - command: - - "/cni-install.sh" - - "--enable-debug=false" - preStop: - exec: - command: - - /cni-uninstall.sh - name: cilium-agent - securityContext: - capabilities: - add: - - NET_ADMIN - - SYS_MODULE - privileged: true - volumeMounts: - - mountPath: /sys/fs/bpf - name: bpf-maps - - mountPath: /var/run/cilium - name: cilium-run - - mountPath: /host/opt/cni/bin - name: cni-path - - mountPath: /host/etc/cni/net.d - name: etc-cni-netd - - mountPath: /var/lib/cilium/clustermesh - name: clustermesh-secrets - readOnly: true - - mountPath: /tmp/cilium/config-map - name: cilium-config-path - readOnly: true - # Needed to be able to load kernel modules - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - mountPath: /run/xtables.lock - name: xtables-lock - hostNetwork: true - initContainers: - - command: - - /init-container.sh - env: - - name: CILIUM_ALL_STATE - valueFrom: - configMapKeyRef: - key: clean-cilium-state - name: cilium-config - optional: true - - name: CILIUM_BPF_STATE - valueFrom: - configMapKeyRef: - key: clean-cilium-bpf-state - name: cilium-config - optional: true - - name: CILIUM_WAIT_BPF_MOUNT - valueFrom: - configMapKeyRef: - key: wait-bpf-mount - name: cilium-config - optional: true - image: "{{ .CiliumImage }}" - imagePullPolicy: IfNotPresent - name: clean-cilium-state - securityContext: - capabilities: - add: - - NET_ADMIN - privileged: true - volumeMounts: - - mountPath: /sys/fs/bpf - name: bpf-maps - mountPropagation: HostToContainer - - mountPath: /var/run/cilium - name: cilium-run - resources: - requests: - cpu: 100m - memory: 100Mi - restartPolicy: Always - priorityClassName: system-node-critical - serviceAccount: cilium - serviceAccountName: cilium - terminationGracePeriodSeconds: 1 - tolerations: - - operator: Exists - volumes: - # To keep state between restarts / upgrades - - hostPath: - path: /var/run/cilium - type: DirectoryOrCreate - name: cilium-run - # To keep state between restarts / upgrades for bpf maps - - hostPath: - path: /sys/fs/bpf - type: DirectoryOrCreate - name: bpf-maps - # To install cilium cni plugin in the host - - hostPath: - path: /opt/cni/bin - type: DirectoryOrCreate - name: cni-path - # To install cilium cni configuration in the host - - hostPath: - path: /etc/cni/net.d - type: DirectoryOrCreate - name: etc-cni-netd - # To be able to load kernel modules - - hostPath: - path: /lib/modules - name: lib-modules - # To access iptables concurrently with other processes (e.g. kube-proxy) - - hostPath: - path: /run/xtables.lock - type: FileOrCreate - name: xtables-lock - # To read the clustermesh configuration - - name: clustermesh-secrets - secret: - defaultMode: 420 - optional: true - secretName: cilium-clustermesh - # To read the configuration from the config map - - configMap: - name: cilium-config - name: cilium-config-path - updateStrategy: - rollingUpdate: - maxUnavailable: 2 - type: RollingUpdate ---- -# Source: cilium/charts/operator/templates/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - io.cilium/app: operator - name: cilium-operator - name: cilium-operator - namespace: kube-system -spec: - # We support HA mode only for Kubernetes version > 1.14 - # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go - # for more details. - replicas: 2 - selector: - matchLabels: - io.cilium/app: operator - name: cilium-operator - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 1 - type: RollingUpdate - template: - metadata: - annotations: - labels: - io.cilium/app: operator - name: cilium-operator - spec: - # In HA mode, cilium-operator pods must not be scheduled on the same - # node as they will clash with each other. - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - podAffinityTerm: - labelSelector: - matchExpressions: - - key: io.cilium/app - operator: In - values: - - operator - topologyKey: kubernetes.io/hostname - weight: 100 - containers: - - args: - - --config-dir=/tmp/cilium/config-map - - --debug=$(CILIUM_DEBUG) - command: - - cilium-operator-generic - env: - - name: K8S_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: CILIUM_K8S_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: CILIUM_DEBUG - valueFrom: - configMapKeyRef: - key: debug - name: cilium-config - optional: true - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - key: AWS_ACCESS_KEY_ID - name: cilium-aws - optional: true - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - key: AWS_SECRET_ACCESS_KEY - name: cilium-aws - optional: true - - name: AWS_DEFAULT_REGION - valueFrom: - secretKeyRef: - key: AWS_DEFAULT_REGION - name: cilium-aws - optional: true - image: "{{ .OperatorGenericImage }}" - imagePullPolicy: IfNotPresent - name: cilium-operator - livenessProbe: - httpGet: - host: '127.0.0.1' - path: /healthz - port: 9234 - scheme: HTTP - initialDelaySeconds: 60 - periodSeconds: 10 - timeoutSeconds: 3 - volumeMounts: - - mountPath: /tmp/cilium/config-map - name: cilium-config-path - readOnly: true - hostNetwork: true - restartPolicy: Always - priorityClassName: system-cluster-critical - serviceAccount: cilium-operator - serviceAccountName: cilium-operator - tolerations: - - operator: Exists - volumes: - # To read the configuration from the config map - - configMap: - name: cilium-config - name: cilium-config-path - `))) diff --git a/cmd/kk/pkg/plugins/network/templates/flannel.go b/cmd/kk/pkg/plugins/network/templates/flannel.go deleted file mode 100644 index 560b1682e..000000000 --- a/cmd/kk/pkg/plugins/network/templates/flannel.go +++ /dev/null @@ -1,504 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var FlannelPSP = template.Must(template.New("network-plugin.yaml").Parse( - dedent.Dedent(`--- -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: psp.flannel.unprivileged - annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default - seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default - apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default - apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default -spec: - privileged: false - volumes: - - configMap - - secret - - emptyDir - - hostPath - allowedHostPaths: - - pathPrefix: "/etc/cni/net.d" - - pathPrefix: "/etc/kube-flannel" - - pathPrefix: "/run/flannel" - readOnlyRootFilesystem: false - # Users and groups - runAsUser: - rule: RunAsAny - supplementalGroups: - rule: RunAsAny - fsGroup: - rule: RunAsAny - # Privilege Escalation - allowPrivilegeEscalation: false - defaultAllowPrivilegeEscalation: false - # Capabilities - allowedCapabilities: ['NET_ADMIN'] - defaultAddCapabilities: [] - requiredDropCapabilities: [] - # Host namespaces - hostPID: false - hostIPC: false - hostNetwork: true - hostPorts: - - min: 0 - max: 65535 - # SELinux - seLinux: - # SELinux is unused in CaaSP - rule: 'RunAsAny' ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - labels: - k8s-app: flannel - name: flannel -rules: - - apiGroups: ['extensions'] - resources: ['podsecuritypolicies'] - verbs: ['use'] - resourceNames: ['psp.flannel.unprivileged'] - - apiGroups: - - "" - resources: - - pods - verbs: - - get - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - - apiGroups: - - networking.k8s.io - resources: - - clustercidrs - verbs: - - list - - watch ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - labels: - k8s-app: flannel - name: flannel -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: flannel -subjects: -- kind: ServiceAccount - name: flannel - namespace: kube-system ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: flannel - namespace: kube-system ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: kube-flannel-cfg - namespace: kube-system - labels: - tier: node - app: flannel - k8s-app: flannel -data: - cni-conf.json: | - { - "name": "cbr0", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "flannel", - "delegate": { - "hairpinMode": true, - "isDefaultGateway": true - } - }, - { - "type": "portmap", - "capabilities": { - "portMappings": true - } - } - ] - } - net-conf.json: | - { - "Network": "{{ .KubePodsCIDR }}", - "Backend": { - "Type": "{{ .BackendMode }}" - } - } ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-flannel-ds - namespace: kube-system - labels: - tier: node - app: flannel - k8s-app: flannel -spec: - selector: - matchLabels: - app: flannel - k8s-app: flannel - template: - metadata: - labels: - tier: node - app: flannel - k8s-app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/os - operator: In - values: - - linux - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - priorityClassName: system-node-critical - serviceAccountName: flannel - initContainers: - - name: install-cni-plugin - args: - - -f - - /flannel - - /opt/cni/bin/flannel - command: - - cp - image: {{ .FlannelPluginImage }} - volumeMounts: - - mountPath: /opt/cni/bin - name: cni-plugin - - name: install-cni - image: {{ .FlannelImage }} - command: - - cp - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - containers: - - name: kube-flannel - image: {{ .FlannelImage }} - command: - - /opt/bin/flanneld - args: - - --ip-masq - - --kube-subnet-mgr - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_ADMIN", "NET_RAW"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: run - mountPath: /run/flannel - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - - mountPath: /run/xtables.lock - name: xtables-lock - volumes: - - name: run - hostPath: - path: /run/flannel - - name: cni-plugin - hostPath: - path: /opt/cni/bin - - name: cni - hostPath: - path: /etc/cni/net.d - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: FileOrCreate - - name: flannel-cfg - configMap: - name: kube-flannel-cfg - - `))) - -var FlannelPS = template.Must(template.New("network-plugin.yaml").Parse( - dedent.Dedent(`--- -apiVersion: v1 -kind: Namespace -metadata: - name: kube-flannel - labels: - pod-security.kubernetes.io/enforce: privileged ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - labels: - k8s-app: flannel - name: flannel -rules: - - apiGroups: - - "" - resources: - - pods - verbs: - - get - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - - apiGroups: - - networking.k8s.io - resources: - - clustercidrs - verbs: - - list - - watch ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - labels: - k8s-app: flannel - name: flannel -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: flannel -subjects: -- kind: ServiceAccount - name: flannel - namespace: kube-flannel ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: flannel - namespace: kube-flannel ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: kube-flannel-cfg - namespace: kube-flannel - labels: - tier: node - app: flannel - k8s-app: flannel -data: - cni-conf.json: | - { - "name": "cbr0", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "flannel", - "delegate": { - "hairpinMode": true, - "isDefaultGateway": true - } - }, - { - "type": "portmap", - "capabilities": { - "portMappings": true - } - } - ] - } - net-conf.json: | - { - "Network": "{{ .KubePodsCIDR }}", - "Backend": { - "Type": "{{ .BackendMode }}" - } - } ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-flannel-ds - namespace: kube-flannel - labels: - tier: node - app: flannel - k8s-app: flannel -spec: - selector: - matchLabels: - app: flannel - k8s-app: flannel - template: - metadata: - labels: - tier: node - app: flannel - k8s-app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/os - operator: In - values: - - linux - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - priorityClassName: system-node-critical - serviceAccountName: flannel - initContainers: - - name: install-cni-plugin - args: - - -f - - /flannel - - /opt/cni/bin/flannel - command: - - cp - image: {{ .FlannelPluginImage }} - volumeMounts: - - mountPath: /opt/cni/bin - name: cni-plugin - - name: install-cni - image: {{ .FlannelImage }} - command: - - cp - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - containers: - - name: kube-flannel - image: {{ .FlannelImage }} - command: - - /opt/bin/flanneld - args: - - --ip-masq - - --kube-subnet-mgr - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_ADMIN", "NET_RAW"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: run - mountPath: /run/flannel - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - - mountPath: /run/xtables.lock - name: xtables-lock - volumes: - - name: run - hostPath: - path: /run/flannel - - name: cni-plugin - hostPath: - path: /opt/cni/bin - - name: cni - hostPath: - path: /etc/cni/net.d - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: FileOrCreate - - name: flannel-cfg - configMap: - name: kube-flannel-cfg - - `))) diff --git a/cmd/kk/pkg/plugins/network/templates/hybridnet.go b/cmd/kk/pkg/plugins/network/templates/hybridnet.go deleted file mode 100644 index 5a39e9cb2..000000000 --- a/cmd/kk/pkg/plugins/network/templates/hybridnet.go +++ /dev/null @@ -1,79 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" - "github.com/lithammer/dedent" - "text/template" -) - -var HybridnetNetworks = template.Must(template.New("hybridnet-networks.yaml").Funcs(utils.FuncMap).Parse( - dedent.Dedent(` -{{- range $index, $network := .Networks }} ---- -apiVersion: networking.alibaba.com/v1 -kind: Network -metadata: - name: {{ $network.Name }} -spec: -{{- if $network.NetID }} - netID: {{ $network.NetID }} -{{- end }} - type: {{ $network.Type }} -{{- if $network.Mode }} - mode: {{ $network.Mode }} -{{- end }} -{{- if $network.NodeSelector }} - nodeSelector: -{{ toYaml $network.NodeSelector | indent 4 }} -{{- end }} - -{{- range $network.Subnets }} ---- -apiVersion: networking.alibaba.com/v1 -kind: Subnet -metadata: - name: {{ .Name }} -spec: - network: {{ $network.Name }} -{{- if .NetID }} - netID: {{ .NetID }} -{{- end }} - range: - version: "4" - cidr: "{{ .CIDR }}" -{{- if .Gateway }} - gateway: "{{ .Gateway }}" -{{- end }} -{{- if .Start}} - start: "{{ .Start }}" -{{- end}} -{{- if .End}} - end: "{{ .End }}" -{{- end }} -{{- if .ReservedIPs }} - reservedIPs: -{{ toYaml .ReservedIPs | indent 4 }} -{{- end }} -{{- if .ExcludeIPs }} - excludeIPs: -{{ toYaml .ExcludeIPs | indent 4 }} -{{- end }} -{{- end }} -{{- end }} - `))) diff --git a/cmd/kk/pkg/plugins/network/templates/kubectl_ko.go b/cmd/kk/pkg/plugins/network/templates/kubectl_ko.go deleted file mode 100644 index 4a51a9a3e..000000000 --- a/cmd/kk/pkg/plugins/network/templates/kubectl_ko.go +++ /dev/null @@ -1,922 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -var KubectlKo = template.Must(template.New("kubectl-ko").Parse( - dedent.Dedent(`#!/bin/bash -set -euo pipefail - -KUBE_OVN_NS=kube-system -WITHOUT_KUBE_PROXY=false -OVN_NB_POD= -OVN_SB_POD= -KUBE_OVN_VERSION= -REGISTRY="kubeovn" - -showHelp(){ - echo "kubectl ko {subcommand} [option...]" - echo "Available Subcommands:" - echo " [nb|sb] [status|kick|backup|dbstatus|restore] ovn-db operations show cluster status, kick stale server, backup database, get db consistency status or restore ovn nb db when met 'inconsistent data' error" - echo " nbctl [ovn-nbctl options ...] invoke ovn-nbctl" - echo " sbctl [ovn-sbctl options ...] invoke ovn-sbctl" - echo " vsctl {nodeName} [ovs-vsctl options ...] invoke ovs-vsctl on the specified node" - echo " ofctl {nodeName} [ovs-ofctl options ...] invoke ovs-ofctl on the specified node" - echo " dpctl {nodeName} [ovs-dpctl options ...] invoke ovs-dpctl on the specified node" - echo " appctl {nodeName} [ovs-appctl options ...] invoke ovs-appctl on the specified node" - echo " tcpdump {namespace/podname} [tcpdump options ...] capture pod traffic" - echo " trace {namespace/podname} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp or udp port] trace ovn microflow of specific packet" - echo " diagnose {all|node} [nodename] diagnose connectivity of all nodes or a specific node" - echo " tuning {install-fastpath|local-install-fastpath|remove-fastpath|install-stt|local-install-stt|remove-stt} {centos7|centos8}} [kernel-devel-version] deploy kernel optimisation components to the system" - echo " reload restart all kube-ovn components" - echo " env-check check the environment configuration" -} - -# usage: ipv4_to_hex 192.168.0.1 -ipv4_to_hex(){ - printf "%02x" ${1//./ } -} - -# convert hex to dec (portable version) -hex2dec(){ - for i in $(echo "$@"); do - printf "%d\n" "$(( 0x$i ))" - done -} - -# https://github.com/chmduquesne/wg-ip -# usage: expand_ipv6 2001::1 -expand_ipv6(){ - local ip=$1 - - # prepend 0 if we start with : - echo $ip | grep -qs "^:" && ip="0${ip}" - - # expand :: - if echo $ip | grep -qs "::"; then - local colons=$(echo $ip | sed 's/[^:]//g') - local missing=$(echo ":::::::::" | sed "s/$colons//") - local expanded=$(echo $missing | sed 's/:/:0/g') - ip=$(echo $ip | sed "s/::/$expanded/") - fi - - local blocks=$(echo $ip | grep -o "[0-9a-f]\+") - set $blocks - - printf "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n" \ - $(hex2dec $@) -} - -# convert an IPv6 address to bytes -ipv6_bytes(){ - for x in $(expand_ipv6 $1 | tr ':' ' '); do - printf "%d %d " $((0x$x >> 8 & 0xff)) $((0x$x & 0xff)) - done - echo -} - -# usage: ipIsInCidr 192.168.0.1 192.168.0.0/24 -# return: 0 for true, 1 for false -ipIsInCidr(){ - local ip=$1 - local cidr=$2 - - if [[ $ip =~ .*:.* ]]; then - # IPv6 - cidr=${cidr#*,} - local network=${cidr%/*} - local prefix=${cidr#*/} - local ip_bytes=($(ipv6_bytes $ip)) - local network_bytes=($(ipv6_bytes $network)) - for ((i=0; i<${#ip_bytes[*]}; i++)); do - if [ ${ip_bytes[$i]} -eq ${network_bytes[$i]} ]; then - continue - fi - - if [ $((($i+1)*8)) -le $prefix ]; then - return 1 - fi - if [ $(($i*8)) -ge $prefix ]; then - return 0 - fi - if [ $((($i+1)*8)) -le $prefix ]; then - return 1 - fi - - local bits=$(($prefix-$i*8)) - local mask=$((0xff<<$bits & 0xff)) - # TODO: check whether the IP is network/broadcast address - if [ $((${ip_bytes[$i]} & $mask)) -ne ${network_bytes[$i]} ]; then - return 1 - fi - done - - return 0 - fi - - # IPv4 - cidr=${cidr%,*} - local network=${cidr%/*} - local prefix=${cidr#*/} - local ip_hex=$(ipv4_to_hex $ip) - local ip_dec=$((0x$ip_hex)) - local network_hex=$(ipv4_to_hex $network) - local network_dec=$((0x$network_hex)) - local broadcast_dec=$(($network_dec + 2**(32-$prefix) - 1)) - # TODO: check whether the IP is network/broadcast address - if [ $ip_dec -gt $network_dec -a $ip_dec -lt $broadcast_dec ]; then - return 0 - fi - - return 1 -} - -tcpdump(){ - namespacedPod="$1"; shift - namespace=$(echo "$namespacedPod" | cut -d "/" -f1) - podName=$(echo "$namespacedPod" | cut -d "/" -f2) - if [ "$podName" = "$namespacedPod" ]; then - namespace="default" - fi - - nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName}) - hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork}) - - if [ -z "$nodeName" ]; then - echo "Pod $namespacedPod not exists on any node" - exit 1 - fi - - ovnCni=$(kubectl get pod -n $KUBE_OVN_NS -l app=kube-ovn-cni -o 'jsonpath={.items[?(@.spec.nodeName=="'$nodeName'")].metadata.name}') - if [ -z "$ovnCni" ]; then - echo "kube-ovn-cni not exist on node $nodeName" - exit 1 - fi - - if [ "$hostNetwork" = "true" ]; then - set -x - kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- tcpdump -nn "$@" - else - nicName=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading --columns=name find interface external-ids:iface-id="$podName"."$namespace" | tr -d '\r') - if [ -z "$nicName" ]; then - echo "nic doesn't exist on node $nodeName" - exit 1 - fi - podNicType=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/pod_nic_type}) - podNetNs=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading get interface "$nicName" external-ids:pod_netns | tr -d '\r' | sed -e 's/^"//' -e 's/"$//') - set -x - if [ "$podNicType" = "internal-port" ]; then - kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" tcpdump -nn -i "$nicName" "$@" - else - kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" tcpdump -nn -i eth0 "$@" - fi - fi -} - -trace(){ - namespacedPod="$1" - namespace=$(echo "$namespacedPod" | cut -d "/" -f1) - podName=$(echo "$namespacedPod" | cut -d "/" -f2) - if [ "$podName" = "$namespacedPod" ]; then - namespace="default" - fi - - dst="$2" - if [ -z "$dst" ]; then - echo "need a target ip address" - exit 1 - fi - - hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork}) - if [ "$hostNetwork" = "true" ]; then - echo "Can not trace host network pod" - exit 1 - fi - - af="4" - nw="nw" - proto="" - if [[ "$dst" =~ .*:.* ]]; then - af="6" - nw="ipv6" - proto="6" - fi - - podIPs=($(kubectl get pod "$podName" -n "$namespace" -o jsonpath="{.status.podIPs[*].ip}")) - if [ ${#podIPs[@]} -eq 0 ]; then - podIPs=($(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/ip_address} | sed 's/,/ /g')) - if [ ${#podIPs[@]} -eq 0 ]; then - echo "pod address not ready" - exit 1 - fi - fi - - podIP="" - for ip in ${podIPs[@]}; do - if [ "$af" = "4" ]; then - if [[ ! "$ip" =~ .*:.* ]]; then - podIP=$ip - break - fi - elif [[ "$ip" =~ .*:.* ]]; then - podIP=$ip - break - fi - done - - if [ -z "$podIP" ]; then - echo "Pod $namespacedPod has no IPv$af address" - exit 1 - fi - - nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName}) - ovnCni=$(kubectl get pod -n $KUBE_OVN_NS -l app=kube-ovn-cni -o 'jsonpath={.items[?(@.spec.nodeName=="'$nodeName'")].metadata.name}') - if [ -z "$ovnCni" ]; then - echo "No kube-ovn-cni Pod running on node $nodeName" - exit 1 - fi - - ls=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_switch}) - if [ -z "$ls" ]; then - echo "pod address not ready" - exit 1 - fi - - local cidr=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/cidr}) - mac=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address}) - - dstMac="" - if echo "$3" | grep -qE '^([[:xdigit:]]{1,2}:){5}[[:xdigit:]]{1,2}$'; then - dstMac=$3 - shift - elif ipIsInCidr $dst $cidr; then - set +o pipefail - if [ $af -eq 4 ]; then - dstMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=addresses list logical_switch_port | grep -w "$(echo $dst | tr . '\.')" | awk '{print $1}') - else - dstMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=addresses list logical_switch_port | grep -i " $dst\$" | awk '{print $1}') - fi - set -o pipefail - fi - - if [ -z "$dstMac" ]; then - vlan=$(kubectl get subnet "$ls" -o jsonpath={.spec.vlan}) - logicalGateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.logicalGateway}) - if [ ! -z "$vlan" -a "$logicalGateway" != "true" ]; then - gateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.gateway}) - if [[ "$gateway" =~ .*,.* ]]; then - if [ "$af" = "4" ]; then - gateway=${gateway%%,*} - else - gateway=${gateway##*,} - fi - fi - - nicName=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading --columns=name find interface external-ids:iface-id="$podName"."$namespace" | tr -d '\r') - if [ -z "$nicName" ]; then - echo "failed to find ovs interface for Pod namespacedPod on node $nodeName" - exit 1 - fi - - podNicType=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/pod_nic_type}) - podNetNs=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading get interface "$nicName" external-ids:pod_netns | tr -d '\r' | sed -e 's/^"//' -e 's/"$//') - if [ "$podNicType" != "internal-port" ]; then - interface=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-vsctl --format=csv --data=bare --no-heading --columns=name find interface external_id:iface-id="$podName"."$namespace") - peer=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ip link show $interface | grep -oE "^[0-9]+:\\s$interface@if[0-9]+" | awk -F @ '{print $2}') - peerIndex=${peer//if/} - peer=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" ip link show type veth | grep "^$peerIndex:" | awk -F @ '{print $1}') - nicName=$(echo $peer | awk '{print $2}') - fi - - set +o pipefail - master=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" ip link show $nicName | grep -Eo '\smaster\s\w+\s' | awk '{print $2}') - set -o pipefail - if [ ! -z "$master" ]; then - echo "Error: Pod nic $nicName is a slave of $master, please set the destination mac address." - exit 1 - fi - - if [[ "$gateway" =~ .*:.* ]]; then - cmd="ndisc6 -q $gateway $nicName" - output=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" ndisc6 -q "$gateway" "$nicName") - else - cmd="arping -c3 -C1 -i1 -I $nicName $gateway" - output=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" arping -c3 -C1 -i1 -I "$nicName" "$gateway") - fi - - if [ $? -ne 0 ]; then - echo "Error: failed to execute '$cmd' in Pod's netns" - exit 1 - fi - - dstMac=$(echo "$output" | grep -oE '([[:xdigit:]]{1,2}:){5}[[:xdigit:]]{1,2}') - fi - fi - - if [ -z "$dstMac" ]; then - echo "Using the gateway mac address as destination" - lr=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_router}) - if [ -z "$lr" ]; then - lr=$(kubectl get subnet "$ls" -o jsonpath={.spec.vpc}) - fi - dstMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=mac find logical_router_port name="$lr"-"$ls" | tr -d '\r') - fi - - if [ -z "$dstMac" ]; then - echo "failed to get destination mac" - exit 1 - fi - - lsp="$podName.$namespace" - lspUUID=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=_uuid find logical_switch_port name="$lsp") - if [ -z "$lspUUID" ]; then - echo "Notice: LSP $lsp does not exist" - fi - vmOwner=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath='{.metadata.ownerReferences[?(@.kind=="VirtualMachineInstance")].name}') - if [ ! -z "$vmOwner" ]; then - lsp="$vmOwner.$namespace" - fi - - if [ -z "$lsp" ]; then - echo "failed to get LSP of Pod $namespace/$podName" - exit 1 - fi - - type="$3" - case $type in - icmp) - set -x - kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$lsp\" && ip.ttl == 64 && icmp && eth.src == $mac && ip$af.src == $podIP && eth.dst == $dstMac && ip$af.dst == $dst" - ;; - tcp|udp) - set -x - kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$lsp\" && ip.ttl == 64 && eth.src == $mac && ip$af.src == $podIP && eth.dst == $dstMac && ip$af.dst == $dst && $type.src == 10000 && $type.dst == $4" - ;; - *) - echo "type $type not supported" - echo "kubectl ko trace {namespace/podname} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp or udp port]" - exit 1 - ;; - esac - - set +x - echo "--------" - echo "Start OVS Tracing" - echo "" - echo "" - - inPort=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-vsctl --format=csv --data=bare --no-heading --columns=ofport find interface external_id:iface-id="$podName"."$namespace") - case $type in - icmp) - set -x - kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,icmp$proto,nw_ttl=64,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$dstMac" - ;; - tcp|udp) - set -x - kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,$type$proto,nw_ttl=64,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$dstMac,${type}_src=1000,${type}_dst=$4" - ;; - *) - echo "type $type not supported" - echo "kubectl ko trace {namespace/podname} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp or udp port]" - exit 1 - ;; - esac -} - -xxctl(){ - subcommand="$1"; shift - nodeName="$1"; shift - kubectl get no "$nodeName" > /dev/null - ovsPod=$(kubectl get pod -n $KUBE_OVN_NS -l app=ovs -o 'jsonpath={.items[?(@.spec.nodeName=="'$nodeName'")].metadata.name}') - if [ -z "$ovsPod" ]; then - echo "ovs pod doesn't exist on node $nodeName" - exit 1 - fi - kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-$subcommand "$@" -} - -checkLeader(){ - component="$1"; shift - set +o pipefail - count=$(kubectl get ep ovn-$component -n $KUBE_OVN_NS -o yaml | grep ip | wc -l) - set -o pipefail - if [ $count -eq 0 ]; then - echo "no ovn-$component exists !!" - exit 1 - fi - - if [ $count -gt 1 ]; then - echo "ovn-$component has more than one leader !!" - exit 1 - fi - - echo "ovn-$component leader check ok" -} - -diagnose(){ - kubectl get crd vpcs.kubeovn.io - kubectl get crd vpc-nat-gateways.kubeovn.io - kubectl get crd subnets.kubeovn.io - kubectl get crd ips.kubeovn.io - kubectl get crd vlans.kubeovn.io - kubectl get crd provider-networks.kubeovn.io - set +eu - if ! kubectl get svc kube-dns -n kube-system ; then - echo "Warning: kube-dns doesn't exist, maybe there is coredns service." - fi - set -eu - kubectl get svc kubernetes -n default - kubectl get sa -n kube-system ovn - kubectl get clusterrole system:ovn - kubectl get clusterrolebinding ovn - - kubectl get no -o wide - kubectl ko nbctl show - kubectl ko nbctl lr-policy-list ovn-cluster - kubectl ko nbctl lr-route-list ovn-cluster - kubectl ko nbctl ls-lb-list ovn-default - kubectl ko nbctl list address_set - kubectl ko nbctl list acl - kubectl ko sbctl show - - if [ "${WITHOUT_KUBE_PROXY}" = "false" ]; then - checkKubeProxy - fi - - checkDeployment ovn-central - checkDeployment kube-ovn-controller - checkDaemonSet kube-ovn-cni - checkDaemonSet ovs-ovn - checkDeployment coredns - - checkLeader nb - checkLeader sb - checkLeader northd - - type="$1" - case $type in - all) - echo "### kube-ovn-controller recent log" - set +e - kubectl logs -n $KUBE_OVN_NS -l app=kube-ovn-controller --tail=100 | grep E$(date +%m%d) - set -e - echo "" - pingers=$(kubectl -n $KUBE_OVN_NS get po --no-headers -o custom-columns=NAME:.metadata.name -l app=kube-ovn-pinger) - for pinger in $pingers - do - nodeName=$(kubectl get pod "$pinger" -n "$KUBE_OVN_NS" -o jsonpath={.spec.nodeName}) - echo "### start to diagnose node $nodeName" - echo "#### ovn-controller log:" - kubectl exec -n $KUBE_OVN_NS "$pinger" -- tail /var/log/ovn/ovn-controller.log - echo "" - echo "#### ovs-vswitchd log:" - kubectl exec -n $KUBE_OVN_NS "$pinger" -- tail /var/log/openvswitch/ovs-vswitchd.log - echo "" - echo "#### ovs-vsctl show results:" - kubectl exec -n $KUBE_OVN_NS "$pinger" -- ovs-vsctl show - echo "" - echo "#### pinger diagnose results:" - kubectl exec -n $KUBE_OVN_NS "$pinger" -- /kube-ovn/kube-ovn-pinger --mode=job - echo "### finish diagnose node $nodeName" - echo "" - done - ;; - node) - nodeName="$2" - kubectl get no "$nodeName" > /dev/null - pinger=$(kubectl -n $KUBE_OVN_NS get po -l app=kube-ovn-pinger -o 'jsonpath={.items[?(@.spec.nodeName=="'$nodeName'")].metadata.name}') - if [ ! -n "$pinger" ]; then - echo "Error: No kube-ovn-pinger running on node $nodeName" - exit 1 - fi - echo "### start to diagnose node $nodeName" - echo "#### ovn-controller log:" - kubectl exec -n $KUBE_OVN_NS "$pinger" -- tail /var/log/ovn/ovn-controller.log - echo "" - echo "#### ovs-vswitchd log:" - kubectl exec -n $KUBE_OVN_NS "$pinger" -- tail /var/log/openvswitch/ovs-vswitchd.log - echo "" - kubectl exec -n $KUBE_OVN_NS "$pinger" -- /kube-ovn/kube-ovn-pinger --mode=job - echo "### finish diagnose node $nodeName" - echo "" - ;; - *) - echo "type $type not supported" - echo "kubectl ko diagnose {all|node} [nodename]" - ;; - esac -} - -getOvnCentralPod(){ - NB_POD=$(kubectl get pod -n $KUBE_OVN_NS -l ovn-nb-leader=true | grep ovn-central | head -n 1 | awk '{print $1}') - if [ -z "$NB_POD" ]; then - echo "nb leader not exists" - exit 1 - fi - OVN_NB_POD=$NB_POD - SB_POD=$(kubectl get pod -n $KUBE_OVN_NS -l ovn-sb-leader=true | grep ovn-central | head -n 1 | awk '{print $1}') - if [ -z "$SB_POD" ]; then - echo "nb leader not exists" - exit 1 - fi - OVN_SB_POD=$SB_POD - VERSION=$(kubectl -n kube-system get pods -l ovn-sb-leader=true -o yaml | grep "image: $REGISTRY/kube-ovn:" | head -n 1 | awk -F ':' '{print $3}') - if [ -z "$VERSION" ]; then - echo "kubeovn version not exists" - exit 1 - fi - KUBE_OVN_VERSION=$VERSION -} - -checkDaemonSet(){ - name="$1" - currentScheduled=$(kubectl get ds -n $KUBE_OVN_NS "$name" -o jsonpath={.status.currentNumberScheduled}) - desiredScheduled=$(kubectl get ds -n $KUBE_OVN_NS "$name" -o jsonpath={.status.desiredNumberScheduled}) - available=$(kubectl get ds -n $KUBE_OVN_NS "$name" -o jsonpath={.status.numberAvailable}) - ready=$(kubectl get ds -n $KUBE_OVN_NS "$name" -o jsonpath={.status.numberReady}) - if [ "$currentScheduled" = "$desiredScheduled" ] && [ "$desiredScheduled" = "$available" ] && [ "$available" = "$ready" ]; then - echo "ds $name ready" - else - echo "Error ds $name not ready" - exit 1 - fi -} - -checkDeployment(){ - name="$1" - ready=$(kubectl get deployment -n $KUBE_OVN_NS "$name" -o jsonpath={.status.readyReplicas}) - updated=$(kubectl get deployment -n $KUBE_OVN_NS "$name" -o jsonpath={.status.updatedReplicas}) - desire=$(kubectl get deployment -n $KUBE_OVN_NS "$name" -o jsonpath={.status.replicas}) - available=$(kubectl get deployment -n $KUBE_OVN_NS "$name" -o jsonpath={.status.availableReplicas}) - if [ "$ready" = "$updated" ] && [ "$updated" = "$desire" ] && [ "$desire" = "$available" ]; then - echo "deployment $name ready" - else - echo "Error deployment $name not ready" - exit 1 - fi -} - -checkKubeProxy(){ - if kubectl get ds -n kube-system --no-headers -o custom-columns=NAME:.metadata.name | grep '^kube-proxy$' >/dev/null; then - checkDaemonSet kube-proxy - else - for node in $(kubectl get node --no-headers -o custom-columns=NAME:.metadata.name); do - local pod=$(kubectl get pod -n $KUBE_OVN_NS -l app=kube-ovn-cni -o 'jsonpath={.items[?(@.spec.nodeName=="'$node'")].metadata.name}') - local ip=$(kubectl get pod -n $KUBE_OVN_NS -l app=kube-ovn-cni -o 'jsonpath={.items[?(@.spec.nodeName=="'$node'")].status.podIP}') - local arg="" - if [[ $ip =~ .*:.* ]]; then - arg="g6" - ip="[$ip]" - fi - healthResult=$(kubectl -n $KUBE_OVN_NS exec $pod -- curl -s${arg} -m 3 -w %{http_code} http://$ip:10256/healthz -o /dev/null | grep -v 200 || true) - if [ -n "$healthResult" ]; then - echo "$node kube-proxy's health check failed" - exit 1 - fi - done - fi - echo "kube-proxy ready" -} - -dbtool(){ - suffix=$(date +%m%d%H%M%s) - component="$1"; shift - action="$1"; shift - case $component in - nb) - case $action in - status) - kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound - kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl ovsdb-server/get-db-storage-status OVN_Northbound - ;; - kick) - kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/kick OVN_Northbound "$1" - ;; - backup) - kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /etc/ovn/ovnnb_db.$suffix.backup /etc/ovn/ovnnb_db.db - kubectl cp $KUBE_OVN_NS/$OVN_NB_POD:/etc/ovn/ovnnb_db.$suffix.backup $(pwd)/ovnnb_db.$suffix.backup - kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /etc/ovn/ovnnb_db.$suffix.backup - echo "backup ovn-$component db to $(pwd)/ovnnb_db.$suffix.backup" - ;; - dbstatus) - kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-appctl -t /var/run/ovn/ovnnb_db.ctl ovsdb-server/get-db-storage-status OVN_Northbound - ;; - restore) - # set ovn-central replicas to 0 - replicas=$(kubectl get deployment -n $KUBE_OVN_NS ovn-central -o jsonpath={.spec.replicas}) - kubectl scale deployment -n $KUBE_OVN_NS ovn-central --replicas=0 - echo "ovn-central original replicas is $replicas" - - # backup ovn-nb db - declare nodeIpArray - declare podNameArray - declare nodeIps - - if [[ $(kubectl get deployment -n kube-system ovn-central -o jsonpath='{.spec.template.spec.containers[0].env[1]}') =~ "NODE_IPS" ]]; then - nodeIpVals=$(kubectl get deployment -n kube-system ovn-central -o jsonpath='{.spec.template.spec.containers[0].env[1].value}') - nodeIps=(${nodeIpVals//,/ }) - else - nodeIps=$(kubectl get node -lkube-ovn/role=master -o wide | grep -v "INTERNAL-IP" | awk '{print $6}') - fi - firstIP=${nodeIps[0]} - podNames=$(kubectl get pod -n $KUBE_OVN_NS | grep ovs-ovn | awk '{print $1}') - echo "first nodeIP is $firstIP" - - i=0 - for nodeIp in ${nodeIps[@]} - do - for pod in $podNames - do - hostip=$(kubectl get pod -n $KUBE_OVN_NS $pod -o jsonpath={.status.hostIP}) - if [ $nodeIp = $hostip ]; then - nodeIpArray[$i]=$nodeIp - podNameArray[$i]=$pod - i=$(expr $i + 1) - echo "ovs-ovn pod on node $nodeIp is $pod" - break - fi - done - done - - echo "backup nb db file" - kubectl exec -it -n $KUBE_OVN_NS ${podNameArray[0]} -- ovsdb-tool cluster-to-standalone /etc/ovn/ovnnb_db_standalone.db /etc/ovn/ovnnb_db.db - - # mv all db files - for pod in ${podNameArray[@]} - do - kubectl exec -it -n $KUBE_OVN_NS $pod -- mv /etc/ovn/ovnnb_db.db /tmp - kubectl exec -it -n $KUBE_OVN_NS $pod -- mv /etc/ovn/ovnsb_db.db /tmp - done - - # restore db and replicas - echo "restore nb db file, operate in pod ${podNameArray[0]}" - kubectl exec -it -n $KUBE_OVN_NS ${podNameArray[0]} -- mv /etc/ovn/ovnnb_db_standalone.db /etc/ovn/ovnnb_db.db - kubectl scale deployment -n $KUBE_OVN_NS ovn-central --replicas=$replicas - echo "finish restore nb db file and ovn-central replicas" - - echo "recreate ovs-ovn pods" - kubectl delete pod -n $KUBE_OVN_NS -l app=ovs - ;; - *) - echo "unknown action $action" - esac - ;; - sb) - case $action in - status) - kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/status OVN_Southbound - kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl ovsdb-server/get-db-storage-status OVN_Southbound - ;; - kick) - kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/kick OVN_Southbound "$1" - ;; - backup) - kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /etc/ovn/ovnsb_db.$suffix.backup /etc/ovn/ovnsb_db.db - kubectl cp $KUBE_OVN_NS/$OVN_SB_POD:/etc/ovn/ovnsb_db.$suffix.backup $(pwd)/ovnsb_db.$suffix.backup - kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /etc/ovn/ovnsb_db.$suffix.backup - echo "backup ovn-$component db to $(pwd)/ovnsb_db.$suffix.backup" - ;; - dbstatus) - kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-appctl -t /var/run/ovn/ovnsb_db.ctl ovsdb-server/get-db-storage-status OVN_Southbound - ;; - restore) - echo "restore cmd is only used for nb db" - ;; - *) - echo "unknown action $action" - esac - ;; - *) - echo "unknown subcommand $component" - esac -} - -tuning(){ - action="$1"; shift - sys="$1"; shift - case $action in - install-fastpath) - case $sys in - centos7) - docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp/:/tmp/ $REGISTRY/centos7-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh centos install" - while [ ! -f /tmp/kube_ovn_fastpath.ko ]; - do - sleep 1 - done - for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); - do - kubectl cp /tmp/kube_ovn_fastpath.ko kube-system/"$i":/tmp/ - done - ;; - centos8) - docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp/:/tmp/ $REGISTRY/centos8-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh centos install" - while [ ! -f /tmp/kube_ovn_fastpath.ko ]; - do - sleep 1 - done - for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); - do - kubectl cp /tmp/kube_ovn_fastpath.ko kube-system/"$i":/tmp/ - done - ;; - *) - echo "unknown system $sys" - esac - ;; - local-install-fastpath) - case $sys in - centos7) - # shellcheck disable=SC2145 - docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp:/tmp $REGISTRY/centos7-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh centos local-install $@" - for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); - do - kubectl cp /tmp/kube_ovn_fastpath.ko kube-system/"$i":/tmp/ - done - ;; - centos8) - # shellcheck disable=SC2145 - docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp:/tmp $REGISTRY/centos8-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh centos local-install $@" - for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); - do - kubectl cp /tmp/kube_ovn_fastpath.ko kube-system/"$i":/tmp/ - done - ;; - *) - echo "unknown system $sys" - esac - ;; - remove-fastpath) - case $sys in - centos) - for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); - do - kubectl -n kube-system exec "$i" -- rm -f /tmp/kube_ovn_fastpath.ko - done - ;; - *) - echo "unknown system $sys" - esac - ;; - install-stt) - case $sys in - centos7) - # shellcheck disable=SC2145 - docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp:/tmp $REGISTRY/centos7-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh stt install" - for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); - do - for k in /tmp/*.rpm; do - kubectl cp "$k" kube-system/"$i":/tmp/ - done - done - ;; - centos8) - # shellcheck disable=SC2145 - docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp:/tmp $REGISTRY/centos8-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh stt install" - for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); - do - for k in /tmp/*.rpm; do - kubectl cp "$k" kube-system/"$i":/tmp/ - done - done - ;; - *) - echo "unknown system $sys" - esac - ;; - local-install-stt) - case $sys in - centos7) - # shellcheck disable=SC2145 - docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp:/tmp $REGISTRY/centos7-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh stt local-install $@" - for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); - do - for k in /tmp/*.rpm; do - kubectl cp "$k" kube-system/"$i":/tmp/ - done - done - ;; - centos8) - # shellcheck disable=SC2145 - docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp:/tmp $REGISTRY/centos8-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh stt local-install $@" - for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); - do - for k in /tmp/*.rpm; do - kubectl cp "$k" kube-system/"$i":/tmp/ - done - done - ;; - *) - echo "unknown system $sys" - esac - ;; - remove-stt) - case $sys in - centos) - for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); - do - kubectl -n kube-system exec "$i" -- rm -f /tmp/openvswitch-kmod*.rpm - done - ;; - *) - echo "unknown system $sys" - esac - ;; - *) - echo "unknown action $action" - esac -} - -reload(){ - kubectl delete pod -n kube-system -l app=ovn-central - kubectl rollout status deployment/ovn-central -n kube-system - kubectl delete pod -n kube-system -l app=ovs - kubectl delete pod -n kube-system -l app=kube-ovn-controller - kubectl rollout status deployment/kube-ovn-controller -n kube-system - kubectl delete pod -n kube-system -l app=kube-ovn-cni - kubectl rollout status daemonset/kube-ovn-cni -n kube-system - kubectl delete pod -n kube-system -l app=kube-ovn-pinger - kubectl rollout status daemonset/kube-ovn-pinger -n kube-system - kubectl delete pod -n kube-system -l app=kube-ovn-monitor - kubectl rollout status deployment/kube-ovn-monitor -n kube-system -} - -env-check(){ - set +e - - KUBE_OVN_NS=kube-system - podNames=$(kubectl get pod --no-headers -n $KUBE_OVN_NS | grep kube-ovn-cni | awk '{print $1}') - for pod in $podNames - do - nodeName=$(kubectl get pod $pod -n $KUBE_OVN_NS -o jsonpath={.spec.nodeName}) - echo "************************************************" - echo "Start environment check for Node $nodeName" - echo "************************************************" - kubectl exec -it -n $KUBE_OVN_NS $pod -c cni-server -- bash /kube-ovn/env-check.sh - done -} - -if [ $# -lt 1 ]; then - showHelp - exit 0 -else - subcommand="$1"; shift -fi - -getOvnCentralPod - -case $subcommand in - nbctl) - kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl "$@" - ;; - sbctl) - kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-sbctl "$@" - ;; - vsctl|ofctl|dpctl|appctl) - xxctl "$subcommand" "$@" - ;; - nb|sb) - dbtool "$subcommand" "$@" - ;; - tcpdump) - tcpdump "$@" - ;; - trace) - trace "$@" - ;; - diagnose) - diagnose "$@" - ;; - reload) - reload - ;; - tuning) - tuning "$@" - ;; - env-check) - env-check - ;; - *) - showHelp - exit 1 - ;; -esac -`))) diff --git a/cmd/kk/pkg/plugins/network/templates/kubeovn.go b/cmd/kk/pkg/plugins/network/templates/kubeovn.go deleted file mode 100644 index 16fad503c..000000000 --- a/cmd/kk/pkg/plugins/network/templates/kubeovn.go +++ /dev/null @@ -1,2780 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -var ( - KubeOvnCrd = template.Must(template.New("kube-ovn-crd.yaml").Parse( - dedent.Dedent(`--- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: vpc-nat-gateways.kubeovn.io -spec: - group: kubeovn.io - names: - plural: vpc-nat-gateways - singular: vpc-nat-gateway - shortNames: - - vpc-nat-gw - kind: VpcNatGateway - listKind: VpcNatGatewayList - scope: Cluster - versions: - - additionalPrinterColumns: - - jsonPath: .spec.vpc - name: Vpc - type: string - - jsonPath: .spec.subnet - name: Subnet - type: string - - jsonPath: .spec.lanIp - name: LanIP - type: string - name: v1 - served: true - storage: true - schema: - openAPIV3Schema: - type: object - properties: - spec: - type: object - properties: - lanIp: - type: string - subnet: - type: string - vpc: - type: string - selector: - type: array - items: - type: string ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: iptables-eips.kubeovn.io -spec: - group: kubeovn.io - names: - plural: iptables-eips - singular: iptables-eip - shortNames: - - eip - kind: IptablesEIP - listKind: IptablesEIPList - scope: Cluster - versions: - - name: v1 - served: true - storage: true - subresources: - status: {} - additionalPrinterColumns: - - jsonPath: .status.ip - name: IP - type: string - - jsonPath: .spec.macAddress - name: Mac - type: string - - jsonPath: .status.nat - name: Nat - type: string - - jsonPath: .spec.natGwDp - name: NatGwDp - type: string - - jsonPath: .status.ready - name: Ready - type: boolean - schema: - openAPIV3Schema: - type: object - properties: - status: - type: object - properties: - ready: - type: boolean - ip: - type: string - nat: - type: string - redo: - type: string - conditions: - type: array - items: - type: object - properties: - type: - type: string - status: - type: string - reason: - type: string - message: - type: string - lastUpdateTime: - type: string - lastTransitionTime: - type: string - spec: - type: object - properties: - v4ip: - type: string - v6ip: - type: string - macAddress: - type: string - natGwDp: - type: string ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: iptables-fip-rules.kubeovn.io -spec: - group: kubeovn.io - names: - plural: iptables-fip-rules - singular: iptables-fip-rule - shortNames: - - fip - kind: IptablesFIPRule - listKind: IptablesFIPRuleList - scope: Cluster - versions: - - name: v1 - served: true - storage: true - subresources: - status: {} - additionalPrinterColumns: - - jsonPath: .spec.eip - name: Eip - type: string - - jsonPath: .status.v4ip - name: V4ip - type: string - - jsonPath: .spec.internalIp - name: InternalIp - type: string - - jsonPath: .status.v6ip - name: V6ip - type: string - - jsonPath: .status.ready - name: Ready - type: boolean - - jsonPath: .status.natGwDp - name: NatGwDp - type: string - schema: - openAPIV3Schema: - type: object - properties: - status: - type: object - properties: - ready: - type: boolean - v4ip: - type: string - v6ip: - type: string - natGwDp: - type: string - redo: - type: string - conditions: - type: array - items: - type: object - properties: - type: - type: string - status: - type: string - reason: - type: string - message: - type: string - lastUpdateTime: - type: string - lastTransitionTime: - type: string - spec: - type: object - properties: - eip: - type: string - internalIp: - type: string ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: iptables-dnat-rules.kubeovn.io -spec: - group: kubeovn.io - names: - plural: iptables-dnat-rules - singular: iptables-dnat-rule - shortNames: - - dnat - kind: IptablesDnatRule - listKind: IptablesDnatRuleList - scope: Cluster - versions: - - name: v1 - served: true - storage: true - subresources: - status: {} - additionalPrinterColumns: - - jsonPath: .spec.eip - name: Eip - type: string - - jsonPath: .spec.protocol - name: Protocol - type: string - - jsonPath: .status.v4ip - name: V4ip - type: string - - jsonPath: .status.v6ip - name: V6ip - type: string - - jsonPath: .spec.internalIp - name: InternalIp - type: string - - jsonPath: .spec.externalPort - name: ExternalPort - type: string - - jsonPath: .spec.internalPort - name: InternalPort - type: string - - jsonPath: .status.natGwDp - name: NatGwDp - type: string - - jsonPath: .status.ready - name: Ready - type: boolean - schema: - openAPIV3Schema: - type: object - properties: - status: - type: object - properties: - ready: - type: boolean - v4ip: - type: string - v6ip: - type: string - natGwDp: - type: string - redo: - type: string - conditions: - type: array - items: - type: object - properties: - type: - type: string - status: - type: string - reason: - type: string - message: - type: string - lastUpdateTime: - type: string - lastTransitionTime: - type: string - spec: - type: object - properties: - eip: - type: string - externalPort: - type: string - protocol: - type: string - internalIp: - type: string - internalPort: - type: string ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: iptables-snat-rules.kubeovn.io -spec: - group: kubeovn.io - names: - plural: iptables-snat-rules - singular: iptables-snat-rule - shortNames: - - snat - kind: IptablesSnatRule - listKind: IptablesSnatRuleList - scope: Cluster - versions: - - name: v1 - served: true - storage: true - subresources: - status: {} - additionalPrinterColumns: - - jsonPath: .spec.eip - name: EIP - type: string - - jsonPath: .status.v4ip - name: V4ip - type: string - - jsonPath: .status.v6ip - name: V6ip - type: string - - jsonPath: .spec.internalCIDR - name: InternalCIDR - type: string - - jsonPath: .status.natGwDp - name: NatGwDp - type: string - - jsonPath: .status.ready - name: Ready - type: boolean - schema: - openAPIV3Schema: - type: object - properties: - status: - type: object - properties: - ready: - type: boolean - v4ip: - type: string - v6ip: - type: string - natGwDp: - type: string - redo: - type: string - conditions: - type: array - items: - type: object - properties: - type: - type: string - status: - type: string - reason: - type: string - message: - type: string - lastUpdateTime: - type: string - lastTransitionTime: - type: string - spec: - type: object - properties: - eip: - type: string - internalCIDR: - type: string ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: vpcs.kubeovn.io -spec: - group: kubeovn.io - versions: - - additionalPrinterColumns: - - jsonPath: .status.standby - name: Standby - type: boolean - - jsonPath: .status.subnets - name: Subnets - type: string - - jsonPath: .spec.namespaces - name: Namespaces - type: string - name: v1 - schema: - openAPIV3Schema: - properties: - spec: - properties: - namespaces: - items: - type: string - type: array - staticRoutes: - items: - properties: - policy: - type: string - cidr: - type: string - nextHopIP: - type: string - type: object - type: array - policyRoutes: - items: - properties: - priority: - type: integer - action: - type: string - match: - type: string - nextHopIP: - type: string - type: object - type: array - vpcPeerings: - items: - properties: - remoteVpc: - type: string - localConnectIP: - type: string - type: object - type: array - type: object - status: - properties: - conditions: - items: - properties: - lastTransitionTime: - type: string - lastUpdateTime: - type: string - message: - type: string - reason: - type: string - status: - type: string - type: - type: string - type: object - type: array - default: - type: boolean - defaultLogicalSwitch: - type: string - router: - type: string - standby: - type: boolean - subnets: - items: - type: string - type: array - vpcPeerings: - items: - type: string - type: array - tcpLoadBalancer: - type: string - tcpSessionLoadBalancer: - type: string - udpLoadBalancer: - type: string - udpSessionLoadBalancer: - type: string - type: object - type: object - served: true - storage: true - subresources: - status: {} - names: - kind: Vpc - listKind: VpcList - plural: vpcs - shortNames: - - vpc - singular: vpc - scope: Cluster ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: ips.kubeovn.io -spec: - group: kubeovn.io - versions: - - name: v1 - served: true - storage: true - additionalPrinterColumns: - - name: V4IP - type: string - jsonPath: .spec.v4IpAddress - - name: V6IP - type: string - jsonPath: .spec.v6IpAddress - - name: Mac - type: string - jsonPath: .spec.macAddress - - name: Node - type: string - jsonPath: .spec.nodeName - - name: Subnet - type: string - jsonPath: .spec.subnet - schema: - openAPIV3Schema: - type: object - properties: - spec: - type: object - properties: - podName: - type: string - namespace: - type: string - subnet: - type: string - attachSubnets: - type: array - items: - type: string - nodeName: - type: string - ipAddress: - type: string - v4IpAddress: - type: string - v6IpAddress: - type: string - attachIps: - type: array - items: - type: string - macAddress: - type: string - attachMacs: - type: array - items: - type: string - containerID: - type: string - podType: - type: string - scope: Cluster - names: - plural: ips - singular: ip - kind: IP - shortNames: - - ip ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: vips.kubeovn.io -spec: - group: kubeovn.io - names: - plural: vips - singular: vip - shortNames: - - vip - kind: Vip - listKind: VipList - scope: Cluster - versions: - - name: v1 - served: true - storage: true - additionalPrinterColumns: - - name: V4IP - type: string - jsonPath: .spec.v4ip - - name: PV4IP - type: string - jsonPath: .spec.parentV4ip - - name: Mac - type: string - jsonPath: .spec.macAddress - - name: PMac - type: string - jsonPath: .spec.ParentMac - - name: V6IP - type: string - jsonPath: .spec.v6ip - - name: PV6IP - type: string - jsonPath: .spec.parentV6ip - - name: Subnet - type: string - jsonPath: .spec.subnet - - jsonPath: .status.ready - name: Ready - type: boolean - schema: - openAPIV3Schema: - type: object - properties: - status: - type: object - properties: - ready: - type: boolean - v4ip: - type: string - v6ip: - type: string - mac: - type: string - pv4ip: - type: string - pv6ip: - type: string - pmac: - type: string - conditions: - type: array - items: - type: object - properties: - type: - type: string - status: - type: string - reason: - type: string - message: - type: string - lastUpdateTime: - type: string - lastTransitionTime: - type: string - spec: - type: object - properties: - namespace: - type: string - subnet: - type: string - attachSubnets: - type: array - items: - type: string - v4ip: - type: string - macAddress: - type: string - v6ip: - type: string - parentV4ip: - type: string - parentMac: - type: string - parentV6ip: - type: string ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: subnets.kubeovn.io -spec: - group: kubeovn.io - versions: - - name: v1 - served: true - storage: true - subresources: - status: {} - additionalPrinterColumns: - - name: Provider - type: string - jsonPath: .spec.provider - - name: Vpc - type: string - jsonPath: .spec.vpc - - name: Protocol - type: string - jsonPath: .spec.protocol - - name: CIDR - type: string - jsonPath: .spec.cidrBlock - - name: Private - type: boolean - jsonPath: .spec.private - - name: NAT - type: boolean - jsonPath: .spec.natOutgoing - - name: Default - type: boolean - jsonPath: .spec.default - - name: GatewayType - type: string - jsonPath: .spec.gatewayType - - name: V4Used - type: number - jsonPath: .status.v4usingIPs - - name: V4Available - type: number - jsonPath: .status.v4availableIPs - - name: V6Used - type: number - jsonPath: .status.v6usingIPs - - name: V6Available - type: number - jsonPath: .status.v6availableIPs - - name: ExcludeIPs - type: string - jsonPath: .spec.excludeIps - schema: - openAPIV3Schema: - type: object - properties: - status: - type: object - properties: - v4availableIPs: - type: number - v4usingIPs: - type: number - v6availableIPs: - type: number - v6usingIPs: - type: number - activateGateway: - type: string - dhcpV4OptionsUUID: - type: string - dhcpV6OptionsUUID: - type: string - conditions: - type: array - items: - type: object - properties: - type: - type: string - status: - type: string - reason: - type: string - message: - type: string - lastUpdateTime: - type: string - lastTransitionTime: - type: string - spec: - type: object - properties: - vpc: - type: string - default: - type: boolean - protocol: - type: string - enum: - - IPv4 - - IPv6 - - Dual - cidrBlock: - type: string - namespaces: - type: array - items: - type: string - gateway: - type: string - provider: - type: string - excludeIps: - type: array - items: - type: string - vips: - type: array - items: - type: string - gatewayType: - type: string - allowSubnets: - type: array - items: - type: string - gatewayNode: - type: string - natOutgoing: - type: boolean - externalEgressGateway: - type: string - policyRoutingPriority: - type: integer - minimum: 1 - maximum: 32765 - policyRoutingTableID: - type: integer - minimum: 1 - maximum: 2147483647 - not: - enum: - - 252 # compat - - 253 # default - - 254 # main - - 255 # local - private: - type: boolean - vlan: - type: string - logicalGateway: - type: boolean - disableGatewayCheck: - type: boolean - disableInterConnection: - type: boolean - htbqos: - type: string - enableDHCP: - type: boolean - dhcpV4Options: - type: string - dhcpV6Options: - type: string - enableIPv6RA: - type: boolean - ipv6RAConfigs: - type: string - acls: - type: array - items: - type: object - properties: - direction: - type: string - enum: - - from-lport - - to-lport - priority: - type: integer - minimum: 0 - maximum: 32767 - match: - type: string - action: - type: string - enum: - - allow-related - - allow-stateless - - allow - - drop - - reject - scope: Cluster - names: - plural: subnets - singular: subnet - kind: Subnet - shortNames: - - subnet ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: vlans.kubeovn.io -spec: - group: kubeovn.io - versions: - - name: v1 - served: true - storage: true - subresources: - status: {} - schema: - openAPIV3Schema: - type: object - properties: - spec: - type: object - properties: - id: - type: integer - minimum: 0 - maximum: 4095 - provider: - type: string - vlanId: - type: integer - description: Deprecated in favor of id - providerInterfaceName: - type: string - description: Deprecated in favor of provider - required: - - provider - status: - type: object - properties: - subnets: - type: array - items: - type: string - additionalPrinterColumns: - - name: ID - type: string - jsonPath: .spec.id - - name: Provider - type: string - jsonPath: .spec.provider - scope: Cluster - names: - plural: vlans - singular: vlan - kind: Vlan - shortNames: - - vlan ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: provider-networks.kubeovn.io -spec: - group: kubeovn.io - versions: - - name: v1 - served: true - storage: true - subresources: - status: {} - schema: - openAPIV3Schema: - type: object - properties: - metadata: - type: object - properties: - name: - type: string - maxLength: 12 - not: - enum: - - int - - external - spec: - type: object - properties: - defaultInterface: - type: string - maxLength: 15 - pattern: '^[^/\s]+$' - customInterfaces: - type: array - items: - type: object - properties: - interface: - type: string - maxLength: 15 - pattern: '^[^/\s]+$' - nodes: - type: array - items: - type: string - exchangeLinkName: - type: boolean - excludeNodes: - type: array - items: - type: string - required: - - defaultInterface - status: - type: object - properties: - ready: - type: boolean - readyNodes: - type: array - items: - type: string - vlans: - type: array - items: - type: string - conditions: - type: array - items: - type: object - properties: - node: - type: string - type: - type: string - status: - type: string - reason: - type: string - message: - type: string - lastUpdateTime: - type: string - lastTransitionTime: - type: string - additionalPrinterColumns: - - name: DefaultInterface - type: string - jsonPath: .spec.defaultInterface - - name: Ready - type: boolean - jsonPath: .status.ready - scope: Cluster - names: - plural: provider-networks - singular: provider-network - kind: ProviderNetwork - listKind: ProviderNetworkList ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: security-groups.kubeovn.io -spec: - group: kubeovn.io - names: - plural: security-groups - singular: security-group - shortNames: - - sg - kind: SecurityGroup - listKind: SecurityGroupList - scope: Cluster - versions: - - name: v1 - served: true - storage: true - schema: - openAPIV3Schema: - type: object - properties: - spec: - type: object - properties: - ingressRules: - type: array - items: - type: object - properties: - ipVersion: - type: string - protocol: - type: string - priority: - type: integer - remoteType: - type: string - remoteAddress: - type: string - remoteSecurityGroup: - type: string - portRangeMin: - type: integer - portRangeMax: - type: integer - policy: - type: string - egressRules: - type: array - items: - type: object - properties: - ipVersion: - type: string - protocol: - type: string - priority: - type: integer - remoteType: - type: string - remoteAddress: - type: string - remoteSecurityGroup: - type: string - portRangeMin: - type: integer - portRangeMax: - type: integer - policy: - type: string - allowSameGroupTraffic: - type: boolean - status: - type: object - properties: - portGroup: - type: string - allowSameGroupTraffic: - type: boolean - ingressMd5: - type: string - egressMd5: - type: string - ingressLastSyncSuccess: - type: boolean - egressLastSyncSuccess: - type: boolean - subresources: - status: {} - conversion: - strategy: None ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: htbqoses.kubeovn.io -spec: - group: kubeovn.io - versions: - - name: v1 - served: true - storage: true - additionalPrinterColumns: - - name: PRIORITY - type: string - jsonPath: .spec.priority - schema: - openAPIV3Schema: - type: object - properties: - spec: - type: object - properties: - priority: - type: string # Value in range 0 to 4,294,967,295. - scope: Cluster - names: - plural: htbqoses - singular: htbqos - kind: HtbQos - shortNames: - - htbqos -`))) - - OVN = template.Must(template.New("ovn.yaml").Parse( - dedent.Dedent(`--- -{{ if .DpdkMode }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: ovn - namespace: kube-system - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - rbac.authorization.k8s.io/system-only: "true" - name: system:ovn -rules: - - apiGroups: ['policy'] - resources: ['podsecuritypolicies'] - verbs: ['use'] - resourceNames: - - kube-ovn - - apiGroups: - - "kubeovn.io" - resources: - - vpcs - - vpcs/status - - vpc-nat-gateways - - subnets - - subnets/status - - ips - - vips - - vips/status - - vlans - - vlans/status - - provider-networks - - provider-networks/status - - security-groups - - security-groups/status - - htbqoses - - iptables-eips - - iptables-fip-rules - - iptables-dnat-rules - - iptables-snat-rules - - iptables-eips/status - - iptables-fip-rules/status - - iptables-dnat-rules/status - - iptables-snat-rules/status - verbs: - - "*" - - apiGroups: - - "" - resources: - - pods - - pods/exec - - namespaces - - nodes - - configmaps - verbs: - - create - - get - - list - - watch - - patch - - update - - apiGroups: - - "k8s.cni.cncf.io" - resources: - - network-attachment-definitions - verbs: - - create - - delete - - get - - list - - update - - apiGroups: - - "" - - networking.k8s.io - - apps - - extensions - resources: - - networkpolicies - - services - - endpoints - - statefulsets - - daemonsets - - deployments - - deployments/scale - verbs: - - create - - delete - - update - - patch - - get - - list - - watch - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - "*" - - apiGroups: - - "k8s.cni.cncf.io" - resources: - - network-attachment-definitions - verbs: - - create - - delete - - get - - list - - update - - apiGroups: - - "kubevirt.io" - resources: - - virtualmachines - - virtualmachineinstances - verbs: - - get - - list ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: ovn -roleRef: - name: system:ovn - kind: ClusterRole - apiGroup: rbac.authorization.k8s.io -subjects: - - kind: ServiceAccount - name: ovn - namespace: kube-system - ---- -kind: Service -apiVersion: v1 -metadata: - name: ovn-nb - namespace: kube-system -spec: - ports: - - name: ovn-nb - protocol: TCP - port: 6641 - targetPort: 6641 - type: ClusterIP - {{ .SvcYamlIpfamilypolicy }} - selector: - app: ovn-central - ovn-nb-leader: "true" - sessionAffinity: None - ---- -kind: Service -apiVersion: v1 -metadata: - name: ovn-sb - namespace: kube-system -spec: - ports: - - name: ovn-sb - protocol: TCP - port: 6642 - targetPort: 6642 - type: ClusterIP - {{ .SvcYamlIpfamilypolicy }} - selector: - app: ovn-central - ovn-sb-leader: "true" - sessionAffinity: None - ---- -kind: Service -apiVersion: v1 -metadata: - name: ovn-northd - namespace: kube-system -spec: - ports: - - name: ovn-northd - protocol: TCP - port: 6643 - targetPort: 6643 - type: ClusterIP - {{ .SvcYamlIpfamilypolicy }} - selector: - app: ovn-central - ovn-northd-leader: "true" - sessionAffinity: None ---- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: ovn-central - namespace: kube-system - annotations: - kubernetes.io/description: | - OVN components: northd, nb and sb. -spec: - replicas: {{ .Count }} - strategy: - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - type: RollingUpdate - selector: - matchLabels: - app: ovn-central - template: - metadata: - labels: - app: ovn-central - component: network - type: infra - spec: - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - app: ovn-central - topologyKey: kubernetes.io/hostname - priorityClassName: system-cluster-critical - serviceAccountName: ovn - hostNetwork: true - containers: - - name: ovn-central - image: "{{ .KubeovnImage }}" - imagePullPolicy: IfNotPresent - command: ["/kube-ovn/start-db.sh"] - securityContext: - capabilities: - add: ["SYS_NICE"] - env: - - name: ENABLE_SSL - value: "{{ .EnableSSL }}" - - name: NODE_IPS - value: {{ .Address }} - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - resources: - requests: - cpu: 300m - memory: 300Mi - limits: - cpu: 3 - memory: 4Gi - volumeMounts: - - mountPath: /var/run/openvswitch - name: host-run-ovs - - mountPath: /var/run/ovn - name: host-run-ovn - - mountPath: /sys - name: host-sys - readOnly: true - - mountPath: /etc/openvswitch - name: host-config-openvswitch - - mountPath: /etc/ovn - name: host-config-ovn - - mountPath: /var/log/openvswitch - name: host-log-ovs - - mountPath: /var/log/ovn - name: host-log-ovn - - mountPath: /etc/localtime - name: localtime - - mountPath: /var/run/tls - name: kube-ovn-tls - readinessProbe: - exec: - command: - - bash - - /kube-ovn/ovn-healthcheck.sh - periodSeconds: 15 - timeoutSeconds: 45 - livenessProbe: - exec: - command: - - bash - - /kube-ovn/ovn-healthcheck.sh - initialDelaySeconds: 30 - periodSeconds: 15 - failureThreshold: 5 - timeoutSeconds: 45 - nodeSelector: - kubernetes.io/os: "linux" - kube-ovn/role: "master" - volumes: - - name: host-run-ovs - hostPath: - path: /run/openvswitch - - name: host-run-ovn - hostPath: - path: /run/ovn - - name: host-sys - hostPath: - path: /sys - - name: host-config-openvswitch - hostPath: - path: /etc/origin/openvswitch - - name: host-config-ovn - hostPath: - path: /etc/origin/ovn - - name: host-log-ovs - hostPath: - path: /var/log/openvswitch - - name: host-log-ovn - hostPath: - path: /var/log/ovn - - name: localtime - hostPath: - path: /etc/localtime - - name: kube-ovn-tls - secret: - optional: true - secretName: kube-ovn-tls - ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: ovs-ovn - namespace: kube-system - annotations: - kubernetes.io/description: | - This daemon set launches the openvswitch daemon. -spec: - selector: - matchLabels: - app: ovs - updateStrategy: - type: OnDelete - template: - metadata: - labels: - app: ovs - component: network - type: infra - spec: - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - priorityClassName: system-cluster-critical - serviceAccountName: ovn - hostNetwork: true - hostPID: true - containers: - - name: openvswitch - image: "kubeovn/kube-ovn-dpdk:{{ .DpdkVersion }}-{{ .OvnVersion }}" - imagePullPolicy: IfNotPresent - command: ["/kube-ovn/start-ovs-dpdk.sh"] - securityContext: - runAsUser: 0 - privileged: true - env: - - name: ENABLE_SSL - value: "{{ .EnableSSL }}" - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: OVN_DB_IPS - value: {{ .Address }} - volumeMounts: - - mountPath: /var/run/netns - name: host-ns - mountPropagation: HostToContainer - - mountPath: /lib/modules - name: host-modules - readOnly: true - - mountPath: /var/run/openvswitch - name: host-run-ovs - - mountPath: /var/run/ovn - name: host-run-ovn - - mountPath: /sys - name: host-sys - readOnly: true - - mountPath: /etc/cni/net.d - name: cni-conf - - mountPath: /etc/openvswitch - name: host-config-openvswitch - - mountPath: /etc/ovn - name: host-config-ovn - - mountPath: /var/log/openvswitch - name: host-log-ovs - - mountPath: /var/log/ovn - name: host-log-ovn - - mountPath: /opt/ovs-config - name: host-config-ovs - - mountPath: /dev/hugepages - name: hugepage - - mountPath: /etc/localtime - name: localtime - - mountPath: /var/run/tls - name: kube-ovn-tls - readinessProbe: - exec: - command: - - bash - - /kube-ovn/ovs-dpdk-healthcheck.sh - periodSeconds: 5 - timeoutSeconds: 45 - livenessProbe: - exec: - command: - - bash - - /kube-ovn/ovs-dpdk-healthcheck.sh - initialDelaySeconds: 60 - periodSeconds: 5 - failureThreshold: 5 - timeoutSeconds: 45 - resources: - requests: - cpu: 1000m - memory: 2Gi - limits: - cpu: 1000m - memory: 2Gi - hugepages-1Gi: 1Gi - nodeSelector: - kubernetes.io/os: "linux" - ovn.kubernetes.io/ovs_dp_type: "kernel" - volumes: - - name: host-modules - hostPath: - path: /lib/modules - - name: host-run-ovs - hostPath: - path: /run/openvswitch - - name: host-run-ovn - hostPath: - path: /run/ovn - - name: host-sys - hostPath: - path: /sys - - name: host-ns - hostPath: - path: /var/run/netns - - name: cni-conf - hostPath: - path: /etc/cni/net.d - - name: host-config-openvswitch - hostPath: - path: /etc/origin/openvswitch - - name: host-config-ovn - hostPath: - path: /etc/origin/ovn - - name: host-log-ovs - hostPath: - path: /var/log/openvswitch - - name: host-log-ovn - hostPath: - path: /var/log/ovn - - name: host-config-ovs - hostPath: - path: /opt/ovs-config - type: DirectoryOrCreate - - name: hugepage - emptyDir: - medium: HugePages - - name: localtime - hostPath: - path: /etc/localtime - - name: kube-ovn-tls - secret: - optional: true - secretName: kube-ovn-tls -{{ else }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: ovn - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - rbac.authorization.k8s.io/system-only: "true" - name: system:ovn -rules: - - apiGroups: ['policy'] - resources: ['podsecuritypolicies'] - verbs: ['use'] - resourceNames: - - kube-ovn - - apiGroups: - - "kubeovn.io" - resources: - - vpcs - - vpcs/status - - vpc-nat-gateways - - subnets - - subnets/status - - ips - - vips - - vips/status - - vlans - - vlans/status - - provider-networks - - provider-networks/status - - security-groups - - security-groups/status - - htbqoses - - iptables-eips - - iptables-fip-rules - - iptables-dnat-rules - - iptables-snat-rules - - iptables-eips/status - - iptables-fip-rules/status - - iptables-dnat-rules/status - - iptables-snat-rules/status - verbs: - - "*" - - apiGroups: - - "" - resources: - - pods - - pods/exec - - namespaces - - nodes - - configmaps - verbs: - - create - - get - - list - - watch - - patch - - update - - apiGroups: - - "" - - networking.k8s.io - - apps - - extensions - resources: - - networkpolicies - - services - - endpoints - - statefulsets - - daemonsets - - deployments - - deployments/scale - verbs: - - create - - delete - - update - - patch - - get - - list - - watch - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - "*" - - apiGroups: - - "k8s.cni.cncf.io" - resources: - - network-attachment-definitions - verbs: - - create - - delete - - get - - list - - update - - apiGroups: - - "kubevirt.io" - resources: - - virtualmachines - - virtualmachineinstances - verbs: - - get - - list ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: ovn -roleRef: - name: system:ovn - kind: ClusterRole - apiGroup: rbac.authorization.k8s.io -subjects: - - kind: ServiceAccount - name: ovn - namespace: kube-system ---- -kind: Service -apiVersion: v1 -metadata: - name: ovn-nb - namespace: kube-system -spec: - ports: - - name: ovn-nb - protocol: TCP - port: 6641 - targetPort: 6641 - type: ClusterIP - {{ .SvcYamlIpfamilypolicy }} - selector: - app: ovn-central - ovn-nb-leader: "true" - sessionAffinity: None ---- -kind: Service -apiVersion: v1 -metadata: - name: ovn-sb - namespace: kube-system -spec: - ports: - - name: ovn-sb - protocol: TCP - port: 6642 - targetPort: 6642 - type: ClusterIP - {{ .SvcYamlIpfamilypolicy }} - selector: - app: ovn-central - ovn-sb-leader: "true" - sessionAffinity: None ---- -kind: Service -apiVersion: v1 -metadata: - name: ovn-northd - namespace: kube-system -spec: - ports: - - name: ovn-northd - protocol: TCP - port: 6643 - targetPort: 6643 - type: ClusterIP - {{ .SvcYamlIpfamilypolicy }} - selector: - app: ovn-central - ovn-northd-leader: "true" - sessionAffinity: None ---- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: ovn-central - namespace: kube-system - annotations: - kubernetes.io/description: | - OVN components: northd, nb and sb. -spec: - replicas: {{ .Count }} - strategy: - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - type: RollingUpdate - selector: - matchLabels: - app: ovn-central - template: - metadata: - labels: - app: ovn-central - component: network - type: infra - spec: - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - app: ovn-central - topologyKey: kubernetes.io/hostname - priorityClassName: system-cluster-critical - serviceAccountName: ovn - hostNetwork: true - containers: - - name: ovn-central - image: "{{ .KubeovnImage }}" - imagePullPolicy: IfNotPresent - command: ["/kube-ovn/start-db.sh"] - securityContext: - capabilities: - add: ["SYS_NICE"] - env: - - name: ENABLE_SSL - value: "{{ .EnableSSL }}" - - name: NODE_IPS - value: {{ .Address }} - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - resources: - requests: - cpu: 300m - memory: 200Mi - limits: - cpu: 3 - memory: 4Gi - volumeMounts: - - mountPath: /var/run/openvswitch - name: host-run-ovs - - mountPath: /var/run/ovn - name: host-run-ovn - - mountPath: /sys - name: host-sys - readOnly: true - - mountPath: /etc/openvswitch - name: host-config-openvswitch - - mountPath: /etc/ovn - name: host-config-ovn - - mountPath: /var/log/openvswitch - name: host-log-ovs - - mountPath: /var/log/ovn - name: host-log-ovn - - mountPath: /etc/localtime - name: localtime - - mountPath: /var/run/tls - name: kube-ovn-tls - readinessProbe: - exec: - command: - - bash - - /kube-ovn/ovn-healthcheck.sh - periodSeconds: 15 - timeoutSeconds: 45 - livenessProbe: - exec: - command: - - bash - - /kube-ovn/ovn-healthcheck.sh - initialDelaySeconds: 30 - periodSeconds: 15 - failureThreshold: 5 - timeoutSeconds: 45 - nodeSelector: - kubernetes.io/os: "linux" - kube-ovn/role: "master" - volumes: - - name: host-run-ovs - hostPath: - path: /run/openvswitch - - name: host-run-ovn - hostPath: - path: /run/ovn - - name: host-sys - hostPath: - path: /sys - - name: host-config-openvswitch - hostPath: - path: /etc/origin/openvswitch - - name: host-config-ovn - hostPath: - path: /etc/origin/ovn - - name: host-log-ovs - hostPath: - path: /var/log/openvswitch - - name: host-log-ovn - hostPath: - path: /var/log/ovn - - name: localtime - hostPath: - path: /etc/localtime - - name: kube-ovn-tls - secret: - optional: true - secretName: kube-ovn-tls ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: ovs-ovn - namespace: kube-system - annotations: - kubernetes.io/description: | - This daemon set launches the openvswitch daemon. -spec: - selector: - matchLabels: - app: ovs - updateStrategy: - type: OnDelete - template: - metadata: - labels: - app: ovs - component: network - type: infra - spec: - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - priorityClassName: system-cluster-critical - serviceAccountName: ovn - hostNetwork: true - hostPID: true - containers: - - name: openvswitch - image: "{{ .KubeovnImage }}" - imagePullPolicy: IfNotPresent - command: ["/kube-ovn/start-ovs.sh"] - securityContext: - runAsUser: 0 - privileged: true - env: - - name: ENABLE_SSL - value: "{{ .EnableSSL }}" - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: HW_OFFLOAD - value: "{{ .HwOffload }}" - - name: TUNNEL_TYPE - value: "{{ .TunnelType }}" - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: OVN_DB_IPS - value: {{ .Address }} - volumeMounts: - - mountPath: /var/run/netns - name: host-ns - mountPropagation: HostToContainer - - mountPath: /lib/modules - name: host-modules - readOnly: true - - mountPath: /var/run/openvswitch - name: host-run-ovs - - mountPath: /var/run/ovn - name: host-run-ovn - - mountPath: /sys - name: host-sys - readOnly: true - - mountPath: /etc/cni/net.d - name: cni-conf - - mountPath: /etc/openvswitch - name: host-config-openvswitch - - mountPath: /etc/ovn - name: host-config-ovn - - mountPath: /var/log/openvswitch - name: host-log-ovs - - mountPath: /var/log/ovn - name: host-log-ovn - - mountPath: /etc/localtime - name: localtime - - mountPath: /var/run/tls - name: kube-ovn-tls - readinessProbe: - exec: - command: - - bash - - -c - - LOG_ROTATE=true /kube-ovn/ovs-healthcheck.sh - periodSeconds: 5 - timeoutSeconds: 45 - livenessProbe: - exec: - command: - - bash - - /kube-ovn/ovs-healthcheck.sh - initialDelaySeconds: 60 - periodSeconds: 5 - failureThreshold: 5 - timeoutSeconds: 45 - resources: - requests: - cpu: 200m - memory: 200Mi - limits: - cpu: 1000m - memory: 1000Mi - nodeSelector: - kubernetes.io/os: "linux" - volumes: - - name: host-modules - hostPath: - path: /lib/modules - - name: host-run-ovs - hostPath: - path: /run/openvswitch - - name: host-run-ovn - hostPath: - path: /run/ovn - - name: host-sys - hostPath: - path: /sys - - name: host-ns - hostPath: - path: /var/run/netns - - name: cni-conf - hostPath: - path: /etc/cni/net.d - - name: host-config-openvswitch - hostPath: - path: /etc/origin/openvswitch - - name: host-config-ovn - hostPath: - path: /etc/origin/ovn - - name: host-log-ovs - hostPath: - path: /var/log/openvswitch - - name: host-log-ovn - hostPath: - path: /var/log/ovn - - name: localtime - hostPath: - path: /etc/localtime - - name: kube-ovn-tls - secret: - optional: true - secretName: kube-ovn-tls -{{ end }}`))) - - KubeOvn = template.Must(template.New("kube-ovn.yaml").Parse( - dedent.Dedent(`--- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: kube-ovn-controller - namespace: kube-system - annotations: - kubernetes.io/description: | - kube-ovn controller -spec: - replicas: {{ .Count }} - selector: - matchLabels: - app: kube-ovn-controller - strategy: - rollingUpdate: - maxSurge: 0% - maxUnavailable: 100% - type: RollingUpdate - template: - metadata: - labels: - app: kube-ovn-controller - component: network - type: infra - spec: - tolerations: - - effect: NoSchedule - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - app: kube-ovn-controller - topologyKey: kubernetes.io/hostname - priorityClassName: system-cluster-critical - serviceAccountName: ovn - hostNetwork: true - containers: - - name: kube-ovn-controller - image: "{{ .KubeovnImage }}" - imagePullPolicy: IfNotPresent - args: - - /kube-ovn/start-controller.sh - - --default-cidr={{ .PodCIDR }} - - --default-gateway={{ .PodGateway }} - - --default-gateway-check={{ .CheckGateway }} - - --default-logical-gateway={{ .LogicalGateway }} - - --default-exclude-ips={{ .ExcludeIps }} - - --node-switch-cidr={{ .JoinCIDR }} - - --service-cluster-ip-range={{ .SvcCIDR }} - - --network-type={{ .NetworkType }} - - --default-interface-name={{ .VlanInterfaceName }} - - --default-vlan-id={{ .VlanID }} - - --pod-nic-type={{ .PodNicType }} - - --enable-lb={{ .EnableLB }} - - --enable-np={{ .EnableNP }} - - --enable-eip-snat={{ .EnableEipSnat }} - - --enable-external-vpc={{ .EnableExternalVPC }} - - --logtostderr=false - - --alsologtostderr=true - - --log_file=/var/log/kube-ovn/kube-ovn-controller.log - - --log_file_max_size=0 - - --keep-vm-ip=true - env: - - name: ENABLE_SSL - value: "{{ .EnableSSL }}" - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KUBE_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: OVN_DB_IPS - value: {{ .Address }} - volumeMounts: - - mountPath: /etc/localtime - name: localtime - - mountPath: /var/log/kube-ovn - name: kube-ovn-log - - mountPath: /var/run/tls - name: kube-ovn-tls - readinessProbe: - exec: - command: - - /kube-ovn/kube-ovn-controller-healthcheck - periodSeconds: 3 - timeoutSeconds: 45 - livenessProbe: - exec: - command: - - /kube-ovn/kube-ovn-controller-healthcheck - initialDelaySeconds: 300 - periodSeconds: 7 - failureThreshold: 5 - timeoutSeconds: 45 - resources: - requests: - cpu: 200m - memory: 200Mi - limits: - cpu: 1000m - memory: 1Gi - nodeSelector: - kubernetes.io/os: "linux" - volumes: - - name: localtime - hostPath: - path: /etc/localtime - - name: kube-ovn-log - hostPath: - path: /var/log/kube-ovn - - name: kube-ovn-tls - secret: - optional: true - secretName: kube-ovn-tls - ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: kube-ovn-cni - namespace: kube-system - annotations: - kubernetes.io/description: | - This daemon set launches the kube-ovn cni daemon. -spec: - selector: - matchLabels: - app: kube-ovn-cni - template: - metadata: - labels: - app: kube-ovn-cni - component: network - type: infra - spec: - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - priorityClassName: system-cluster-critical - serviceAccountName: ovn - hostNetwork: true - hostPID: true - initContainers: - - name: install-cni - image: "{{ .KubeovnImage }}" - imagePullPolicy: IfNotPresent - command: ["/kube-ovn/install-cni.sh"] - securityContext: - runAsUser: 0 - privileged: true - volumeMounts: - - mountPath: /opt/cni/bin - name: cni-bin - containers: - - name: cni-server - image: "{{ .KubeovnImage }}" - imagePullPolicy: IfNotPresent - command: - - bash - - /kube-ovn/start-cniserver.sh - args: - - --enable-mirror={{ .EnableMirror }} - - --encap-checksum=true - - --service-cluster-ip-range={{ .SvcCIDR }} - - --iface={{ .Iface }} - - --dpdk-tunnel-iface={{ .DpdkTunnelIface }} - - --network-type={{ .TunnelType }} - - --default-interface-name={{ .VlanInterfaceName }} - - --cni-conf-name={{ .CNIConfigPriority }}-kube-ovn.conflist - - --logtostderr=false - - --alsologtostderr=true - - --log_file=/var/log/kube-ovn/kube-ovn-cni.log - - --log_file_max_size=0 - securityContext: - runAsUser: 0 - privileged: true - env: - - name: ENABLE_SSL - value: "{{ .EnableSSL }}" - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: MODULES - value: {{ .Modules }} - - name: RPMS - value: {{ .RPMs }} - volumeMounts: - - name: host-modules - mountPath: /lib/modules - readOnly: true - - name: shared-dir - mountPath: /var/lib/kubelet/pods - - mountPath: /etc/openvswitch - name: systemid - - mountPath: /etc/cni/net.d - name: cni-conf - - mountPath: /run/openvswitch - name: host-run-ovs - mountPropagation: Bidirectional - - mountPath: /run/ovn - name: host-run-ovn - - mountPath: /var/run/netns - name: host-ns - mountPropagation: HostToContainer - - mountPath: /var/log/kube-ovn - name: kube-ovn-log - - mountPath: /var/log/openvswitch - name: host-log-ovs - - mountPath: /var/log/ovn - name: host-log-ovn - - mountPath: /etc/localtime - name: localtime - - mountPath: /tmp - name: tmp - livenessProbe: - failureThreshold: 3 - initialDelaySeconds: 30 - periodSeconds: 7 - successThreshold: 1 - tcpSocket: - port: 10665 - timeoutSeconds: 3 - readinessProbe: - failureThreshold: 3 - initialDelaySeconds: 30 - periodSeconds: 7 - successThreshold: 1 - tcpSocket: - port: 10665 - timeoutSeconds: 3 - resources: - requests: - cpu: 100m - memory: 100Mi - limits: - cpu: 1000m - memory: 1Gi - nodeSelector: - kubernetes.io/os: "linux" - volumes: - - name: host-modules - hostPath: - path: /lib/modules - - name: shared-dir - hostPath: - path: /var/lib/kubelet/pods - - name: systemid - hostPath: - path: /etc/origin/openvswitch - - name: host-run-ovs - hostPath: - path: /run/openvswitch - - name: host-run-ovn - hostPath: - path: /run/ovn - - name: cni-conf - hostPath: - path: /etc/cni/net.d - - name: cni-bin - hostPath: - path: /opt/cni/bin - - name: host-ns - hostPath: - path: /var/run/netns - - name: host-log-ovs - hostPath: - path: /var/log/openvswitch - - name: kube-ovn-log - hostPath: - path: /var/log/kube-ovn - - name: host-log-ovn - hostPath: - path: /var/log/ovn - - name: localtime - hostPath: - path: /etc/localtime - - name: tmp - hostPath: - path: /tmp - ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: kube-ovn-pinger - namespace: kube-system - annotations: - kubernetes.io/description: | - This daemon set launches the openvswitch daemon. -spec: - selector: - matchLabels: - app: kube-ovn-pinger - updateStrategy: - type: RollingUpdate - template: - metadata: - labels: - app: kube-ovn-pinger - component: network - type: infra - spec: - serviceAccountName: ovn - hostPID: true - containers: - - name: pinger - image: "{{ .KubeovnImage }}" - command: - - /kube-ovn/kube-ovn-pinger - args: - - --external-address={{ .PingExternalAddress }} - - --external-dns={{ .PingExternalDNS }} - - --logtostderr=false - - --alsologtostderr=true - - --log_file=/var/log/kube-ovn/kube-ovn-pinger.log - - --log_file_max_size=0 - imagePullPolicy: IfNotPresent - securityContext: - runAsUser: 0 - privileged: false - env: - - name: ENABLE_SSL - value: "{{ .EnableSSL }}" - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: HOST_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - mountPath: /lib/modules - name: host-modules - readOnly: true - - mountPath: /run/openvswitch - name: host-run-ovs - - mountPath: /var/run/openvswitch - name: host-run-ovs - - mountPath: /var/run/ovn - name: host-run-ovn - - mountPath: /sys - name: host-sys - readOnly: true - - mountPath: /etc/openvswitch - name: host-config-openvswitch - - mountPath: /var/log/openvswitch - name: host-log-ovs - - mountPath: /var/log/ovn - name: host-log-ovn - - mountPath: /var/log/kube-ovn - name: kube-ovn-log - - mountPath: /etc/localtime - name: localtime - - mountPath: /var/run/tls - name: kube-ovn-tls - resources: - requests: - cpu: 100m - memory: 100Mi - limits: - cpu: 200m - memory: 400Mi - nodeSelector: - kubernetes.io/os: "linux" - volumes: - - name: host-modules - hostPath: - path: /lib/modules - - name: host-run-ovs - hostPath: - path: /run/openvswitch - - name: host-run-ovn - hostPath: - path: /run/ovn - - name: host-sys - hostPath: - path: /sys - - name: host-config-openvswitch - hostPath: - path: /etc/origin/openvswitch - - name: host-log-ovs - hostPath: - path: /var/log/openvswitch - - name: kube-ovn-log - hostPath: - path: /var/log/kube-ovn - - name: host-log-ovn - hostPath: - path: /var/log/ovn - - name: localtime - hostPath: - path: /etc/localtime - - name: kube-ovn-tls - secret: - optional: true - secretName: kube-ovn-tls ---- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: kube-ovn-monitor - namespace: kube-system - annotations: - kubernetes.io/description: | - Metrics for OVN components: northd, nb and sb. -spec: - replicas: 1 - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 1 - type: RollingUpdate - selector: - matchLabels: - app: kube-ovn-monitor - template: - metadata: - labels: - app: kube-ovn-monitor - component: network - type: infra - spec: - tolerations: - - effect: NoSchedule - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - app: kube-ovn-monitor - topologyKey: kubernetes.io/hostname - priorityClassName: system-cluster-critical - serviceAccountName: ovn - hostNetwork: true - containers: - - name: kube-ovn-monitor - image: "{{ .KubeovnImage }}" - imagePullPolicy: IfNotPresent - command: ["/kube-ovn/start-ovn-monitor.sh"] - securityContext: - runAsUser: 0 - privileged: false - env: - - name: ENABLE_SSL - value: "{{ .EnableSSL }}" - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - resources: - requests: - cpu: 200m - memory: 200Mi - limits: - cpu: 200m - memory: 200Mi - volumeMounts: - - mountPath: /var/run/openvswitch - name: host-run-ovs - - mountPath: /var/run/ovn - name: host-run-ovn - - mountPath: /etc/openvswitch - name: host-config-openvswitch - - mountPath: /etc/ovn - name: host-config-ovn - - mountPath: /var/log/openvswitch - name: host-log-ovs - - mountPath: /var/log/ovn - name: host-log-ovn - - mountPath: /etc/localtime - name: localtime - - mountPath: /var/run/tls - name: kube-ovn-tls - readinessProbe: - exec: - command: - - cat - - /var/run/ovn/ovn-controller.pid - periodSeconds: 10 - timeoutSeconds: 45 - livenessProbe: - exec: - command: - - cat - - /var/run/ovn/ovn-controller.pid - initialDelaySeconds: 30 - periodSeconds: 10 - failureThreshold: 5 - timeoutSeconds: 45 - nodeSelector: - kubernetes.io/os: "linux" - kube-ovn/role: "master" - volumes: - - name: host-run-ovs - hostPath: - path: /run/openvswitch - - name: host-run-ovn - hostPath: - path: /run/ovn - - name: host-config-openvswitch - hostPath: - path: /etc/origin/openvswitch - - name: host-config-ovn - hostPath: - path: /etc/origin/ovn - - name: host-log-ovs - hostPath: - path: /var/log/openvswitch - - name: host-log-ovn - hostPath: - path: /var/log/ovn - - name: localtime - hostPath: - path: /etc/localtime - - name: kube-ovn-tls - secret: - optional: true - secretName: kube-ovn-tls ---- -kind: Service -apiVersion: v1 -metadata: - name: kube-ovn-monitor - namespace: kube-system - labels: - app: kube-ovn-monitor -spec: - ports: - - name: metrics - port: 10661 - type: ClusterIP - {{ .SvcYamlIpfamilypolicy }} - selector: - app: kube-ovn-monitor - sessionAffinity: None ---- -kind: Service -apiVersion: v1 -metadata: - name: kube-ovn-pinger - namespace: kube-system - labels: - app: kube-ovn-pinger -spec: - {{ .SvcYamlIpfamilypolicy }} - selector: - app: kube-ovn-pinger - ports: - - port: 8080 - name: metrics ---- -kind: Service -apiVersion: v1 -metadata: - name: kube-ovn-controller - namespace: kube-system - labels: - app: kube-ovn-controller -spec: - {{ .SvcYamlIpfamilypolicy }} - selector: - app: kube-ovn-controller - ports: - - port: 10660 - name: metrics ---- -kind: Service -apiVersion: v1 -metadata: - name: kube-ovn-cni - namespace: kube-system - labels: - app: kube-ovn-cni -spec: - {{ .SvcYamlIpfamilypolicy }} - selector: - app: kube-ovn-cni - ports: - - port: 10665 - name: metrics`))) -) diff --git a/cmd/kk/pkg/plugins/network/templates/multus.go b/cmd/kk/pkg/plugins/network/templates/multus.go deleted file mode 100644 index 4fa47ee77..000000000 --- a/cmd/kk/pkg/plugins/network/templates/multus.go +++ /dev/null @@ -1,229 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var Multus = template.Must(template.New("multus-network-plugin.yaml").Parse( - dedent.Dedent(` ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: network-attachment-definitions.k8s.cni.cncf.io -spec: - group: k8s.cni.cncf.io - scope: Namespaced - names: - plural: network-attachment-definitions - singular: network-attachment-definition - kind: NetworkAttachmentDefinition - shortNames: - - net-attach-def - versions: - - name: v1 - served: true - storage: true - schema: - openAPIV3Schema: - description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing - Working Group to express the intent for attaching pods to one or more logical or physical - networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec' - type: object - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this represen - tation of an object. Servers should convert recognized schemas to the - latest internal value, and may reject unrecognized values. More info: - https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment' - type: object - properties: - config: - description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration' - type: string ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: multus -rules: - - apiGroups: ["k8s.cni.cncf.io"] - resources: - - '*' - verbs: - - '*' - - apiGroups: - - "" - resources: - - pods - - pods/status - verbs: - - get - - update - - apiGroups: - - "" - - events.k8s.io - resources: - - events - verbs: - - create - - patch - - update ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: multus -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: multus -subjects: -- kind: ServiceAccount - name: multus - namespace: kube-system ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: multus - namespace: kube-system ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: multus-cni-config - namespace: kube-system - labels: - tier: node - app: multus -data: - # NOTE: If you'd prefer to manually apply a configuration file, you may create one here. - # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod - # change the "args" line below from - # - "--multus-conf-file=auto" - # to: - # "--multus-conf-file=/tmp/multus-conf/70-multus.conf" - # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the - # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet. - cni-conf.json: | - { - "name": "multus-cni-network", - "type": "multus", - "capabilities": { - "portMappings": true - }, - "delegates": [ - { - "cniVersion": "0.3.1", - "name": "default-cni-network", - "plugins": [ - { - "type": "flannel", - "name": "flannel.1", - "delegate": { - "isDefaultGateway": true, - "hairpinMode": true - } - }, - { - "type": "portmap", - "capabilities": { - "portMappings": true - } - } - ] - } - ], - "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig" - } ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-multus-ds - namespace: kube-system - labels: - tier: node - app: multus - name: multus -spec: - selector: - matchLabels: - name: multus - updateStrategy: - type: RollingUpdate - template: - metadata: - labels: - tier: node - app: multus - name: multus - spec: - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - serviceAccountName: multus - containers: - - name: kube-multus - image: {{ .MultusImage }} - command: ["/entrypoint.sh"] - args: - - "--multus-conf-file=auto" - - "--cni-version=0.3.1" - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: true - volumeMounts: - - name: cni - mountPath: /host/etc/cni/net.d - - name: cnibin - mountPath: /host/opt/cni/bin - - name: multus-cfg - mountPath: /tmp/multus-conf - terminationGracePeriodSeconds: 10 - volumes: - - name: cni - hostPath: - path: /etc/cni/net.d - - name: cnibin - hostPath: - path: /opt/cni/bin - - name: multus-cfg - configMap: - name: multus-cni-config - items: - - key: cni-conf.json - path: 70-multus.conf -`))) diff --git a/cmd/kk/pkg/plugins/nfd.go b/cmd/kk/pkg/plugins/nfd.go deleted file mode 100644 index 0f53603e7..000000000 --- a/cmd/kk/pkg/plugins/nfd.go +++ /dev/null @@ -1,700 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package plugins - -import ( - "path/filepath" - "text/template" - - "github.com/lithammer/dedent" - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" -) - -// NodeFeatureDiscovery detects hardware features available on each node in a Kubernetes cluster, and advertises those -// features using node labels. - -var ( - NodeFeatureDiscovery = template.Must(template.New("node-feature-discovery.yaml").Parse( - dedent.Dedent(`--- -apiVersion: v1 -kind: Namespace -metadata: - name: node-feature-discovery ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.7.0 - creationTimestamp: null - name: nodefeaturerules.nfd.k8s-sigs.io -spec: - group: nfd.k8s-sigs.io - names: - kind: NodeFeatureRule - listKind: NodeFeatureRuleList - plural: nodefeaturerules - singular: nodefeaturerule - scope: Cluster - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: NodeFeatureRule resource specifies a configuration for feature-based customization of node objects, such as node labeling. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: NodeFeatureRuleSpec describes a NodeFeatureRule. - properties: - rules: - description: Rules is a list of node customization rules. - items: - description: Rule defines a rule for node customization such as labeling. - properties: - labels: - additionalProperties: - type: string - description: Labels to create if the rule matches. - type: object - labelsTemplate: - description: LabelsTemplate specifies a template to expand for dynamically generating multiple labels. Data (after template expansion) must be keys with an optional value ([=]) separated by newlines. - type: string - matchAny: - description: MatchAny specifies a list of matchers one of which must match. - items: - description: MatchAnyElem specifies one sub-matcher of MatchAny. - properties: - matchFeatures: - description: MatchFeatures specifies a set of matcher terms all of which must match. - items: - description: FeatureMatcherTerm defines requirements against one feature set. All requirements (specified as MatchExpressions) are evaluated against each element in the feature set. - properties: - feature: - type: string - matchExpressions: - additionalProperties: - description: "MatchExpression specifies an expression to evaluate against a set of input values. It contains an operator that is applied when matching the input and an array of values that the operator evaluates the input against. \n NB: CreateMatchExpression or MustCreateMatchExpression() should be used for creating new instances. NB: Validate() must be called if Op or Value fields are modified or if a new instance is created from scratch without using the helper functions." - properties: - op: - description: Op is the operator to be applied. - enum: - - In - - NotIn - - InRegexp - - Exists - - DoesNotExist - - Gt - - Lt - - GtLt - - IsTrue - - IsFalse - type: string - value: - description: Value is the list of values that the operand evaluates the input against. Value should be empty if the operator is Exists, DoesNotExist, IsTrue or IsFalse. Value should contain exactly one element if the operator is Gt or Lt and exactly two elements if the operator is GtLt. In other cases Value should contain at least one element. - items: - type: string - type: array - required: - - op - type: object - description: MatchExpressionSet contains a set of MatchExpressions, each of which is evaluated against a set of input values. - type: object - required: - - feature - - matchExpressions - type: object - type: array - required: - - matchFeatures - type: object - type: array - matchFeatures: - description: MatchFeatures specifies a set of matcher terms all of which must match. - items: - description: FeatureMatcherTerm defines requirements against one feature set. All requirements (specified as MatchExpressions) are evaluated against each element in the feature set. - properties: - feature: - type: string - matchExpressions: - additionalProperties: - description: "MatchExpression specifies an expression to evaluate against a set of input values. It contains an operator that is applied when matching the input and an array of values that the operator evaluates the input against. \n NB: CreateMatchExpression or MustCreateMatchExpression() should be used for creating new instances. NB: Validate() must be called if Op or Value fields are modified or if a new instance is created from scratch without using the helper functions." - properties: - op: - description: Op is the operator to be applied. - enum: - - In - - NotIn - - InRegexp - - Exists - - DoesNotExist - - Gt - - Lt - - GtLt - - IsTrue - - IsFalse - type: string - value: - description: Value is the list of values that the operand evaluates the input against. Value should be empty if the operator is Exists, DoesNotExist, IsTrue or IsFalse. Value should contain exactly one element if the operator is Gt or Lt and exactly two elements if the operator is GtLt. In other cases Value should contain at least one element. - items: - type: string - type: array - required: - - op - type: object - description: MatchExpressionSet contains a set of MatchExpressions, each of which is evaluated against a set of input values. - type: object - required: - - feature - - matchExpressions - type: object - type: array - name: - description: Name of the rule. - type: string - vars: - additionalProperties: - type: string - description: Vars is the variables to store if the rule matches. Variables do not directly inflict any changes in the node object. However, they can be referenced from other rules enabling more complex rule hierarchies, without exposing intermediary output values as labels. - type: object - varsTemplate: - description: VarsTemplate specifies a template to expand for dynamically generating multiple variables. Data (after template expansion) must be keys with an optional value ([=]) separated by newlines. - type: string - required: - - name - type: object - type: array - required: - - rules - type: object - required: - - spec - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: nfd-master - namespace: node-feature-discovery ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: nfd-master -rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - patch - - update - - list -- apiGroups: - - topology.node.k8s.io - resources: - - noderesourcetopologies - verbs: - - create - - get - - update -- apiGroups: - - nfd.k8s-sigs.io - resources: - - nodefeaturerules - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: nfd-master -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: nfd-master -subjects: -- kind: ServiceAccount - name: nfd-master - namespace: node-feature-discovery ---- -apiVersion: v1 -data: - nfd-worker.conf: | - #core: - # labelWhiteList: - # noPublish: false - # sleepInterval: 60s - # featureSources: [all] - # labelSources: [all] - # klog: - # addDirHeader: false - # alsologtostderr: false - # logBacktraceAt: - # logtostderr: true - # skipHeaders: false - # stderrthreshold: 2 - # v: 0 - # vmodule: - ## NOTE: the following options are not dynamically run-time configurable - ## and require a nfd-worker restart to take effect after being changed - # logDir: - # logFile: - # logFileMaxSize: 1800 - # skipLogHeaders: false - #sources: - # cpu: - # cpuid: - ## NOTE: whitelist has priority over blacklist - # attributeBlacklist: - # - "BMI1" - # - "BMI2" - # - "CLMUL" - # - "CMOV" - # - "CX16" - # - "ERMS" - # - "F16C" - # - "HTT" - # - "LZCNT" - # - "MMX" - # - "MMXEXT" - # - "NX" - # - "POPCNT" - # - "RDRAND" - # - "RDSEED" - # - "RDTSCP" - # - "SGX" - # - "SSE" - # - "SSE2" - # - "SSE3" - # - "SSE4" - # - "SSE42" - # - "SSSE3" - # attributeWhitelist: - # kernel: - # kconfigFile: "/path/to/kconfig" - # configOpts: - # - "NO_HZ" - # - "X86" - # - "DMI" - # pci: - # deviceClassWhitelist: - # - "0200" - # - "03" - # - "12" - # deviceLabelFields: - # - "class" - # - "vendor" - # - "device" - # - "subsystem_vendor" - # - "subsystem_device" - # usb: - # deviceClassWhitelist: - # - "0e" - # - "ef" - # - "fe" - # - "ff" - # deviceLabelFields: - # - "class" - # - "vendor" - # - "device" - # custom: - # # The following feature demonstrates the capabilities of the matchFeatures - # - name: "my custom rule" - # labels: - # my-ng-feature: "true" - # # matchFeatures implements a logical AND over all matcher terms in the - # # list (i.e. all of the terms, or per-feature matchers, must match) - # matchFeatures: - # - feature: cpu.cpuid - # matchExpressions: - # AVX512F: {op: Exists} - # - feature: cpu.cstate - # matchExpressions: - # enabled: {op: IsTrue} - # - feature: cpu.pstate - # matchExpressions: - # no_turbo: {op: IsFalse} - # scaling_governor: {op: In, value: ["performance"]} - # - feature: cpu.rdt - # matchExpressions: - # RDTL3CA: {op: Exists} - # - feature: cpu.sst - # matchExpressions: - # bf.enabled: {op: IsTrue} - # - feature: cpu.topology - # matchExpressions: - # hardware_multithreading: {op: IsFalse} - # - # - feature: kernel.config - # matchExpressions: - # X86: {op: Exists} - # LSM: {op: InRegexp, value: ["apparmor"]} - # - feature: kernel.loadedmodule - # matchExpressions: - # e1000e: {op: Exists} - # - feature: kernel.selinux - # matchExpressions: - # enabled: {op: IsFalse} - # - feature: kernel.version - # matchExpressions: - # major: {op: In, value: ["5"]} - # minor: {op: Gt, value: ["10"]} - # - # - feature: storage.block - # matchExpressions: - # rotational: {op: In, value: ["0"]} - # dax: {op: In, value: ["0"]} - # - # - feature: network.device - # matchExpressions: - # operstate: {op: In, value: ["up"]} - # speed: {op: Gt, value: ["100"]} - # - # - feature: memory.numa - # matchExpressions: - # node_count: {op: Gt, value: ["2"]} - # - feature: memory.nv - # matchExpressions: - # devtype: {op: In, value: ["nd_dax"]} - # mode: {op: In, value: ["memory"]} - # - # - feature: system.osrelease - # matchExpressions: - # ID: {op: In, value: ["fedora", "centos"]} - # - feature: system.name - # matchExpressions: - # nodename: {op: InRegexp, value: ["^worker-X"]} - # - # - feature: local.label - # matchExpressions: - # custom-feature-knob: {op: Gt, value: ["100"]} - # - # # The following feature demonstrates the capabilities of the matchAny - # - name: "my matchAny rule" - # labels: - # my-ng-feature-2: "my-value" - # # matchAny implements a logical IF over all elements (sub-matchers) in - # # the list (i.e. at least one feature matcher must match) - # matchAny: - # - matchFeatures: - # - feature: kernel.loadedmodule - # matchExpressions: - # driver-module-X: {op: Exists} - # - feature: pci.device - # matchExpressions: - # vendor: {op: In, value: ["8086"]} - # class: {op: In, value: ["0200"]} - # - matchFeatures: - # - feature: kernel.loadedmodule - # matchExpressions: - # driver-module-Y: {op: Exists} - # - feature: usb.device - # matchExpressions: - # vendor: {op: In, value: ["8086"]} - # class: {op: In, value: ["02"]} - # - # # The following features demonstreate label templating capabilities - # - name: "my template rule" - # labelsTemplate: | - # matchFeatures: - # - feature: system.osrelease - # matchExpressions: - # ID: {op: InRegexp, value: ["^open.*"]} - # VERSION_ID.major: {op: In, value: ["13", "15"]} - # - # - name: "my template rule 2" - # matchFeatures: - # - feature: pci.device - # matchExpressions: - # class: {op: InRegexp, value: ["^06"]} - # vendor: ["8086"] - # - feature: cpu.cpuid - # matchExpressions: - # AVX: {op: Exists} - # - # # The following examples demonstrate vars field and back-referencing - # # previous labels and vars - # - name: "my dummy kernel rule" - # labels: - # "my.kernel.feature": "true" - # matchFeatures: - # - feature: kernel.version - # matchExpressions: - # major: {op: Gt, value: ["2"]} - # - # - name: "my dummy rule with no labels" - # vars: - # "my.dummy.var": "1" - # matchFeatures: - # - feature: cpu.cpuid - # matchExpressions: {} - # - # - name: "my rule using backrefs" - # labels: - # "my.backref.feature": "true" - # matchFeatures: - # - feature: rule.matched - # matchExpressions: - # my.kernel.feature: {op: IsTrue} - # my.dummy.var: {op: Gt, value: ["0"]} - # -kind: ConfigMap -metadata: - name: nfd-worker-conf - namespace: node-feature-discovery ---- -apiVersion: v1 -kind: Service -metadata: - name: nfd-master - namespace: node-feature-discovery -spec: - ports: - - port: 8080 - protocol: TCP - selector: - app: nfd-master - type: ClusterIP ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: nfd - name: nfd-master - namespace: node-feature-discovery -spec: - replicas: 1 - selector: - matchLabels: - app: nfd-master - template: - metadata: - labels: - app: nfd-master - spec: - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - preference: - matchExpressions: - - key: node-role.kubernetes.io/master - operator: In - values: - - "" - weight: 1 - - preference: - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: In - values: - - "" - weight: 1 - containers: - - args: [] - command: - - nfd-master - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - image: {{ .NFDImage }} - imagePullPolicy: IfNotPresent - livenessProbe: - exec: - command: - - /usr/bin/grpc_health_probe - - -addr=:8080 - initialDelaySeconds: 10 - periodSeconds: 10 - name: nfd-master - readinessProbe: - exec: - command: - - /usr/bin/grpc_health_probe - - -addr=:8080 - failureThreshold: 10 - initialDelaySeconds: 5 - periodSeconds: 10 - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsNonRoot: true - volumeMounts: [] - serviceAccount: nfd-master - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Equal - value: "" - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Equal - value: "" - volumes: [] ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - app: nfd - name: nfd-worker - namespace: node-feature-discovery -spec: - selector: - matchLabels: - app: nfd-worker - template: - metadata: - labels: - app: nfd-worker - spec: - containers: - - args: - - -server=nfd-master:8080 - command: - - nfd-worker - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - image: {{ .NFDImage }} - imagePullPolicy: IfNotPresent - name: nfd-worker - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsNonRoot: true - volumeMounts: - - mountPath: /host-boot - name: host-boot - readOnly: true - - mountPath: /host-etc/os-release - name: host-os-release - readOnly: true - - mountPath: /host-sys - name: host-sys - readOnly: true - - mountPath: /host-usr/lib - name: host-usr-lib - readOnly: true - - mountPath: /etc/kubernetes/node-feature-discovery/source.d/ - name: source-d - readOnly: true - - mountPath: /etc/kubernetes/node-feature-discovery/features.d/ - name: features-d - readOnly: true - - mountPath: /etc/kubernetes/node-feature-discovery - name: nfd-worker-conf - readOnly: true - dnsPolicy: ClusterFirstWithHostNet - volumes: - - hostPath: - path: /boot - name: host-boot - - hostPath: - path: /etc/os-release - name: host-os-release - - hostPath: - path: /sys - name: host-sys - - hostPath: - path: /usr/lib - name: host-usr-lib - - hostPath: - path: /etc/kubernetes/node-feature-discovery/source.d/ - name: source-d - - hostPath: - path: /etc/kubernetes/node-feature-discovery/features.d/ - name: features-d - - configMap: - name: nfd-worker-conf - name: nfd-worker-conf - - `))) -) - -func DeployNodeFeatureDiscoveryTasks(d *DeployPluginsModule) []task.Interface { - generateNFDManifests := &task.RemoteTask{ - Name: "GenerateNFDManifests", - Desc: "Generate node-feature-discovery manifests", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: &action.Template{ - Template: NodeFeatureDiscovery, - Data: util.Data{ - "NFDImage": images.GetImage(d.Runtime, d.KubeConf, "node-feature-discovery").ImageName(), - }, - Dst: filepath.Join(common.KubeAddonsDir, NodeFeatureDiscovery.Name()), - }, - Parallel: false, - } - - deployNFD := &task.RemoteTask{ - Name: "ApplyNFDManifests", - Desc: "Apply node-feature-discovery manifests", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(ApplyNodeFeatureDiscoveryManifests), - } - - return []task.Interface{ - generateNFDManifests, - deployNFD, - } -} - -type ApplyNodeFeatureDiscoveryManifests struct { - common.KubeAction -} - -func (a *ApplyNodeFeatureDiscoveryManifests) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/addons/node-feature-discovery.yaml", true); err != nil { - return errors.Wrap(errors.WithStack(err), "apply node-feature-discovery manifests failed") - } - return nil -} diff --git a/cmd/kk/pkg/plugins/storage/modules.go b/cmd/kk/pkg/plugins/storage/modules.go deleted file mode 100644 index c46b4d09c..000000000 --- a/cmd/kk/pkg/plugins/storage/modules.go +++ /dev/null @@ -1,80 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package storage - -import ( - "path/filepath" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/storage/templates" -) - -type DeployLocalVolumeModule struct { - common.KubeModule - Skip bool -} - -func (d *DeployLocalVolumeModule) IsSkip() bool { - return d.Skip -} - -func (d *DeployLocalVolumeModule) Init() { - d.Name = "DeployStorageClassModule" - d.Desc = "Deploy cluster storage-class" - - generate := &task.RemoteTask{ - Name: "GenerateOpenEBSManifest", - Desc: "Generate OpenEBS manifest", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(CheckDefaultStorageClass), - }, - Action: &action.Template{ - Template: templates.OpenEBS, - Dst: filepath.Join(common.KubeAddonsDir, templates.OpenEBS.Name()), - Data: util.Data{ - "ProvisionerLocalPVImage": images.GetImage(d.Runtime, d.KubeConf, "provisioner-localpv").ImageName(), - "LinuxUtilsImage": images.GetImage(d.Runtime, d.KubeConf, "linux-utils").ImageName(), - "BasePath": d.KubeConf.Cluster.Storage.OpenEBS.BasePath, - }, - }, - Parallel: true, - } - - deploy := &task.RemoteTask{ - Name: "DeployOpenEBS", - Desc: "Deploy OpenEBS as cluster default StorageClass", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(CheckDefaultStorageClass), - }, - Action: new(DeployLocalVolume), - Parallel: true, - } - - d.Tasks = []task.Interface{ - generate, - deploy, - } -} diff --git a/cmd/kk/pkg/plugins/storage/prepares.go b/cmd/kk/pkg/plugins/storage/prepares.go deleted file mode 100644 index 7294dfad3..000000000 --- a/cmd/kk/pkg/plugins/storage/prepares.go +++ /dev/null @@ -1,48 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package storage - -import ( - "regexp" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" -) - -type CheckDefaultStorageClass struct { - common.KubePrepare -} - -func (c *CheckDefaultStorageClass) PreCheck(runtime connector.Runtime) (bool, error) { - output, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl get sc --no-headers | grep '(default)' | wc -l", false) - if err != nil { - return false, errors.Wrap(errors.WithStack(err), "check default storageClass failed") - } - - reg := regexp.MustCompile(`([\d])`) - defaultStorageClassNum := reg.FindStringSubmatch(output)[0] - if defaultStorageClassNum == "0" { - return true, nil - } - host := runtime.RemoteHost() - logger.Log.Messagef(host.GetName(), "Default storageClass in cluster is not unique!") - return false, nil -} diff --git a/cmd/kk/pkg/plugins/storage/tasks.go b/cmd/kk/pkg/plugins/storage/tasks.go deleted file mode 100644 index 79da86aef..000000000 --- a/cmd/kk/pkg/plugins/storage/tasks.go +++ /dev/null @@ -1,39 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package storage - -import ( - "fmt" - "path/filepath" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type DeployLocalVolume struct { - common.KubeAction -} - -func (d *DeployLocalVolume) Execute(runtime connector.Runtime) error { - cmd := fmt.Sprintf("/usr/local/bin/kubectl apply -f %s", filepath.Join(common.KubeAddonsDir, "local-volume.yaml")) - if _, err := runtime.GetRunner().SudoCmd(cmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "deploy local-volume.yaml failed") - } - return nil -} diff --git a/cmd/kk/pkg/plugins/storage/templates/openebs.go b/cmd/kk/pkg/plugins/storage/templates/openebs.go deleted file mode 100644 index 2523ecee9..000000000 --- a/cmd/kk/pkg/plugins/storage/templates/openebs.go +++ /dev/null @@ -1,177 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -// OpenEBS defines the template of openebs' manifests. -var OpenEBS = template.Must(template.New("local-volume.yaml").Parse( - dedent.Dedent(`--- -#Sample storage classes for OpenEBS Local PV -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: local - annotations: - storageclass.kubesphere.io/supported-access-modes: '["ReadWriteOnce"]' - storageclass.beta.kubernetes.io/is-default-class: "true" - openebs.io/cas-type: local - cas.openebs.io/config: | - - name: StorageType - value: "hostpath" - - name: BasePath - value: "{{ .BasePath }}" -provisioner: openebs.io/local -volumeBindingMode: WaitForFirstConsumer -reclaimPolicy: Delete ---- -# Create Maya Service Account -apiVersion: v1 -kind: ServiceAccount -metadata: - name: openebs-maya-operator - namespace: kube-system ---- -# Define Role that allows operations on K8s pods/deployments -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: openebs-maya-operator -rules: -- apiGroups: ["*"] - resources: ["nodes", "nodes/proxy"] - verbs: ["*"] -- apiGroups: ["*"] - resources: ["namespaces", "services", "pods", "pods/exec", "deployments", "deployments/finalizers", "replicationcontrollers", "replicasets", "events", "endpoints", "configmaps", "secrets", "jobs", "cronjobs"] - verbs: ["*"] -- apiGroups: ["*"] - resources: ["statefulsets", "daemonsets"] - verbs: ["*"] -- apiGroups: ["*"] - resources: ["resourcequotas", "limitranges"] - verbs: ["list", "watch"] -- apiGroups: ["*"] - resources: ["ingresses", "horizontalpodautoscalers", "verticalpodautoscalers", "poddisruptionbudgets", "certificatesigningrequests"] - verbs: ["list", "watch"] -- apiGroups: ["*"] - resources: ["storageclasses", "persistentvolumeclaims", "persistentvolumes"] - verbs: ["*"] -- apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: [ "get", "list", "create", "update", "delete", "patch"] -- apiGroups: ["openebs.io"] - resources: [ "*"] - verbs: ["*"] -- nonResourceURLs: ["/metrics"] - verbs: ["get"] ---- -# Bind the Service Account with the Role Privileges. -# TODO: Check if default account also needs to be there -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: openebs-maya-operator -subjects: -- kind: ServiceAccount - name: openebs-maya-operator - namespace: kube-system -roleRef: - kind: ClusterRole - name: openebs-maya-operator - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: openebs-localpv-provisioner - namespace: kube-system - labels: - name: openebs-localpv-provisioner - openebs.io/component-name: openebs-localpv-provisioner - openebs.io/version: 3.3.0 -spec: - selector: - matchLabels: - name: openebs-localpv-provisioner - openebs.io/component-name: openebs-localpv-provisioner - replicas: 1 - strategy: - type: Recreate - template: - metadata: - labels: - name: openebs-localpv-provisioner - openebs.io/component-name: openebs-localpv-provisioner - openebs.io/version: 3.3.0 - spec: - serviceAccountName: openebs-maya-operator - containers: - - name: openebs-provisioner-hostpath - imagePullPolicy: IfNotPresent - image: {{ .ProvisionerLocalPVImage }} - env: - # OPENEBS_IO_K8S_MASTER enables openebs provisioner to connect to K8s - # based on this address. This is ignored if empty. - # This is supported for openebs provisioner version 0.5.2 onwards - #- name: OPENEBS_IO_K8S_MASTER - # value: "http://10.128.0.12:8080" - # OPENEBS_IO_KUBE_CONFIG enables openebs provisioner to connect to K8s - # based on this config. This is ignored if empty. - # This is supported for openebs provisioner version 0.5.2 onwards - #- name: OPENEBS_IO_KUBE_CONFIG - # value: "/home/ubuntu/.kube/config" - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: OPENEBS_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - # OPENEBS_SERVICE_ACCOUNT provides the service account of this pod as - # environment variable - - name: OPENEBS_SERVICE_ACCOUNT - valueFrom: - fieldRef: - fieldPath: spec.serviceAccountName - - name: OPENEBS_IO_ENABLE_ANALYTICS - value: "true" - - name: OPENEBS_IO_INSTALLER_TYPE - value: "openebs-operator-lite" - - name: OPENEBS_IO_HELPER_IMAGE - value: "{{ .LinuxUtilsImage }}" - # LEADER_ELECTION_ENABLED is used to enable/disable leader election. By default - # leader election is enabled. - #- name: LEADER_ELECTION_ENABLED - # value: "true" - # OPENEBS_IO_IMAGE_PULL_SECRETS environment variable is used to pass the image pull secrets - # to the helper pod launched by local-pv hostpath provisioner - #- name: OPENEBS_IO_IMAGE_PULL_SECRETS - # value: "" - livenessProbe: - exec: - command: - - sh - - -c - - test $(pgrep -c "^provisioner-loc.*") = 1 - initialDelaySeconds: 30 - periodSeconds: 60 - `))) diff --git a/cmd/kk/pkg/registry/docker_registry_config.go b/cmd/kk/pkg/registry/docker_registry_config.go deleted file mode 100644 index d8d79b6db..000000000 --- a/cmd/kk/pkg/registry/docker_registry_config.go +++ /dev/null @@ -1,123 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package registry - -import ( - "encoding/json" - "os" - "path/filepath" - "strings" - - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/runtime" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" -) - -type DockerRegistryEntry struct { - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` - SkipTLSVerify bool `json:"skipTLSVerify,omitempty"` - PlainHTTP bool `json:"plainHTTP,omitempty"` - CertsPath string `json:"certsPath,omitempty"` - // CAFile is an SSL Certificate Authority file used to secure etcd communication. - CAFile string `yaml:"caFile" json:"caFile,omitempty"` - // CertFile is an SSL certification file used to secure etcd communication. - CertFile string `yaml:"certFile" json:"certFile,omitempty"` - // KeyFile is an SSL key file used to secure etcd communication. - KeyFile string `yaml:"keyFile" json:"keyFile,omitempty"` -} - -func DockerRegistryAuthEntries(auths runtime.RawExtension) (entries map[string]*DockerRegistryEntry) { - - if len(auths.Raw) == 0 { - return - } - - err := json.Unmarshal(auths.Raw, &entries) - if err != nil { - logger.Log.Fatalf("Failed to Parse Registry Auths configuration: %v", auths.Raw) - return - } - - for _, v := range entries { - if v.CertsPath != "" { - ca, cert, key, err := LookupCertsFile(v.CertsPath) - if err != nil { - logger.Log.Warningf("Failed to lookup certs file from the specific cert path %s: %s", v.CertsPath, err.Error()) - return - } - v.CAFile = ca - v.CertFile = cert - v.KeyFile = key - } - if v.PlainHTTP { - v.SkipTLSVerify = true - } - } - - return -} - -func LookupCertsFile(path string) (ca string, cert string, key string, err error) { - logger.Log.Debugf("Looking for TLS certificates and private keys in %s", path) - absPath, err := filepath.Abs(path) - if err != nil { - return - } - logger.Log.Debugf("Looking for TLS certificates and private keys in abs path %s", absPath) - entries, err := os.ReadDir(absPath) - if err != nil { - return ca, cert, key, err - } - - for _, f := range entries { - fullPath := filepath.Join(path, f.Name()) - if strings.HasSuffix(f.Name(), ".crt") { - logger.Log.Debugf(" crt: %s", fullPath) - ca = fullPath - } - if strings.HasSuffix(f.Name(), ".cert") { - certName := f.Name() - keyName := certName[:len(certName)-5] + ".key" - logger.Log.Debugf(" cert: %s", fullPath) - if !hasFile(entries, keyName) { - return ca, cert, key, errors.Errorf("missing key %s for client certificate %s. Note that CA certificates should use the extension .crt", keyName, certName) - } - cert = fullPath - } - if strings.HasSuffix(f.Name(), ".key") { - keyName := f.Name() - certName := keyName[:len(keyName)-4] + ".cert" - logger.Log.Debugf(" key: %s", fullPath) - if !hasFile(entries, certName) { - return ca, cert, key, errors.Errorf("missing client certificate %s for key %s", certName, keyName) - } - key = fullPath - } - } - return ca, cert, key, nil -} - -func hasFile(files []os.DirEntry, name string) bool { - for _, f := range files { - if f.Name() == name { - return true - } - } - return false -} diff --git a/cmd/kk/pkg/registry/k3s_registries.go b/cmd/kk/pkg/registry/k3s_registries.go deleted file mode 100644 index aed028835..000000000 --- a/cmd/kk/pkg/registry/k3s_registries.go +++ /dev/null @@ -1,60 +0,0 @@ -package registry - -// Mirror contains the config related to the registry mirror -type Mirror struct { - // Endpoints are endpoints for a namespace. CRI plugin will try the endpoints - // one by one until a working one is found. The endpoint must be a valid url - // with host specified. - // The scheme, host and path from the endpoint URL will be used. - Endpoints []string `toml:"endpoint" yaml:"endpoint"` - - // Rewrites are repository rewrite rules for a namespace. When fetching image resources - // from an endpoint and a key matches the repository via regular expression matching - // it will be replaced with the corresponding value from the map in the resource request. - Rewrites map[string]string `toml:"rewrite" yaml:"rewrite"` -} - -// AuthConfig contains the config related to authentication to a specific registry -type AuthConfig struct { - // Username is the username to login the registry. - Username string `toml:"username" yaml:"username"` - // Password is the password to login the registry. - Password string `toml:"password" yaml:"password"` - // Auth is a base64 encoded string from the concatenation of the username, - // a colon, and the password. - Auth string `toml:"auth" yaml:"auth"` - // IdentityToken is used to authenticate the user and get - // an access token for the registry. - IdentityToken string `toml:"identitytoken" yaml:"identity_token"` -} - -// TLSConfig contains the CA/Cert/Key used for a registry -type TLSConfig struct { - CAFile string `toml:"ca_file" yaml:"ca_file"` - CertFile string `toml:"cert_file" yaml:"cert_file"` - KeyFile string `toml:"key_file" yaml:"key_file"` - InsecureSkipVerify bool `toml:"insecure_skip_verify" yaml:"insecure_skip_verify"` -} - -// Registry is registry settings including mirrors, TLS, and credentials -type Registry struct { - // Mirrors are namespace to mirror mapping for all namespaces. - Mirrors map[string]Mirror `toml:"mirrors" yaml:"mirrors"` - // Configs are configs for each registry. - // The key is the FDQN or IP of the registry. - Configs map[string]RegistryConfig `toml:"configs" yaml:"configs"` - - // Auths are registry endpoint to auth config mapping. The registry endpoint must - // be a valid url with host specified. - // DEPRECATED: Use Configs instead. Remove in containerd 1.4. - Auths map[string]AuthConfig `toml:"auths" yaml:"auths"` -} - -// RegistryConfig contains configuration used to communicate with the registry. -type RegistryConfig struct { - // Auth contains information to authenticate to the registry. - Auth *AuthConfig `toml:"auth" yaml:"auth"` - // TLS is a pair of CA/Cert/Key which then are used when creating the transport - // that communicates with the registry. - TLS *TLSConfig `toml:"tls" yaml:"tls"` -} diff --git a/cmd/kk/pkg/utils/certs/certs.go b/cmd/kk/pkg/utils/certs/certs.go deleted file mode 100644 index e66ea57b9..000000000 --- a/cmd/kk/pkg/utils/certs/certs.go +++ /dev/null @@ -1,333 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package certs - -import ( - "crypto" - cryptorand "crypto/rand" - "crypto/x509" - "crypto/x509/pkix" - "fmt" - "math" - "math/big" - "net" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "k8s.io/apimachinery/pkg/util/sets" - certutil "k8s.io/client-go/util/cert" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" -) - -const ( - // CertificateValidity defines the validity for all the signed certificates generated by kubeadm - CertificateValidity = time.Hour * 24 * 365 * 10 - // CertificateBlockType is a possible value for pem.Block.Type. - CertificateBlockType = "CERTIFICATE" - rsaKeySize = 2048 -) - -// KubekeyCert represents a certificate that Kubeadm will create to function properly. -type KubekeyCert struct { - Name string - LongName string - BaseName string - CAName string - Config CertConfig -} - -// GetConfig returns the definition for the given cert given the provided InitConfiguration -func (k *KubekeyCert) GetConfig(_ *common.KubeConf) (*CertConfig, error) { - - return &k.Config, nil -} - -// CreateFromCA makes and writes a certificate using the given CA cert and key. -func (k *KubekeyCert) CreateFromCA(kubeConf *common.KubeConf, pkiPath string, caCert *x509.Certificate, caKey crypto.Signer) error { - cfg, err := k.GetConfig(kubeConf) - if err != nil { - return errors.Wrapf(err, "couldn't create %q certificate", k.Name) - } - cert, key, err := NewCertAndKey(caCert, caKey, cfg) - if err != nil { - return err - } - err = writeCertificateFilesIfNotExist( - pkiPath, - k.BaseName, - caCert, - cert, - key, - cfg, - ) - - if err != nil { - return errors.Wrapf(err, "failed to write or validate certificate %q", k.Name) - } - - return nil -} - -func GenerateCA(ca *KubekeyCert, pkiPath string, kubeConf *common.KubeConf) error { - - if cert, err := TryLoadCertFromDisk(pkiPath, ca.BaseName); err == nil { - CheckCertificatePeriodValidity(ca.BaseName, cert) - - if _, err := TryLoadKeyFromDisk(pkiPath, ca.BaseName); err == nil { - fmt.Printf("[certs] Using existing %s certificate authority\n", ca.BaseName) - return nil - } - fmt.Printf("[certs] Using existing %s keyless certificate authority\n", ca.BaseName) - return nil - } - - // create the new certificate authority (or use existing) - return CreateCACertAndKeyFiles(ca, pkiPath, kubeConf) - -} - -// CreateCACertAndKeyFiles generates and writes out a given certificate authority. -// The certSpec should be one of the variables from this package. -func CreateCACertAndKeyFiles(certSpec *KubekeyCert, pkiPath string, kubeConf *common.KubeConf) error { - if certSpec.CAName != "" { - return errors.Errorf("this function should only be used for CAs, but cert %s has CA %s", certSpec.Name, certSpec.CAName) - } - - certConfig, err := certSpec.GetConfig(kubeConf) - if err != nil { - return err - } - - caCert, caKey, err := NewCertificateAuthority(certConfig) - if err != nil { - return err - } - - return writeCertificateAuthorityFilesIfNotExist( - pkiPath, - certSpec.BaseName, - caCert, - caKey, - ) -} - -func GenerateCerts(cert *KubekeyCert, caCert *KubekeyCert, pkiPath string, kubeConf *common.KubeConf) error { - // TODO: if using external etcd, skips etcd certificates generation - - if certData, intermediates, err := TryLoadCertChainFromDisk(pkiPath, cert.BaseName); err == nil { - CheckCertificatePeriodValidity(cert.BaseName, certData) - - caCertData, err := TryLoadCertFromDisk(pkiPath, caCert.BaseName) - if err != nil { - return errors.Wrapf(err, "couldn't load CA certificate %s", caCert.Name) - } - - CheckCertificatePeriodValidity(caCert.BaseName, caCertData) - - if err := VerifyCertChain(certData, intermediates, caCertData); err != nil { - return errors.Wrapf(err, "[certs] certificate %s not signed by CA certificate %s", cert.BaseName, caCert.BaseName) - } - - fmt.Printf("[certs] Using existing %s certificate and key on disk\n", cert.BaseName) - return nil - } - - // create the new certificate (or use existing) - return CreateCertAndKeyFilesWithCA(caCert, cert, pkiPath, kubeConf) -} - -// CreateCertAndKeyFilesWithCA loads the given certificate authority from disk, then generates and writes out the given certificate and key. -// The certSpec and caCertSpec should both be one of the variables from this package. -func CreateCertAndKeyFilesWithCA(caCertSpec *KubekeyCert, certSpec *KubekeyCert, pkiPath string, kubeConf *common.KubeConf) error { - if certSpec.CAName != caCertSpec.Name { - return errors.Errorf("expected CAname for %s to be %q, but was %s", certSpec.Name, certSpec.CAName, caCertSpec.Name) - } - - caCert, caKey, err := LoadCertificateAuthority(pkiPath, caCertSpec.BaseName) - if err != nil { - return errors.Wrapf(err, "couldn't load CA certificate %s", caCertSpec.Name) - } - - return certSpec.CreateFromCA(kubeConf, pkiPath, caCert, caKey) -} - -// LoadCertificateAuthority tries to load a CA in the given directory with the given name. -func LoadCertificateAuthority(pkiDir string, baseName string) (*x509.Certificate, crypto.Signer, error) { - // Checks if certificate authority exists in the PKI directory - if !CertOrKeyExist(pkiDir, baseName) { - return nil, nil, errors.Errorf("couldn't load %s certificate authority from %s", baseName, pkiDir) - } - - // Try to load certificate authority .crt and .key from the PKI directory - caCert, caKey, err := TryLoadCertAndKeyFromDisk(pkiDir, baseName) - if err != nil { - return nil, nil, errors.Wrapf(err, "failure loading %s certificate authority", baseName) - } - // Validate period - CheckCertificatePeriodValidity(baseName, caCert) - - // Make sure the loaded CA cert actually is a CA - if !caCert.IsCA { - return nil, nil, errors.Errorf("%s certificate is not a certificate authority", baseName) - } - - return caCert, caKey, nil -} - -// NewCertAndKey creates new certificate and key by passing the certificate authority certificate and key -func NewCertAndKey(caCert *x509.Certificate, caKey crypto.Signer, config *CertConfig) (*x509.Certificate, crypto.Signer, error) { - if len(config.Usages) == 0 { - return nil, nil, errors.New("must specify at least one ExtKeyUsage") - } - - key, err := NewPrivateKey(config.PublicKeyAlgorithm) - if err != nil { - return nil, nil, errors.Wrap(err, "unable to create private key") - } - - cert, err := NewSignedCert(config, key, caCert, caKey, false) - if err != nil { - return nil, nil, errors.Wrap(err, "unable to sign certificate") - } - - return cert, key, nil -} - -// NewSignedCert creates a signed certificate using the given CA certificate and key -func NewSignedCert(cfg *CertConfig, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer, isCA bool) (*x509.Certificate, error) { - serial, err := cryptorand.Int(cryptorand.Reader, new(big.Int).SetInt64(math.MaxInt64)) - if err != nil { - return nil, err - } - if len(cfg.CommonName) == 0 { - return nil, errors.New("must specify a CommonName") - } - - keyUsage := x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature - if isCA { - keyUsage |= x509.KeyUsageCertSign - } - - RemoveDuplicateAltNames(&cfg.AltNames) - - notAfter := time.Now().Add(CertificateValidity).UTC() - if cfg.NotAfter != nil { - notAfter = *cfg.NotAfter - } - - certTmpl := x509.Certificate{ - Subject: pkix.Name{ - CommonName: cfg.CommonName, - Organization: cfg.Organization, - }, - DNSNames: cfg.AltNames.DNSNames, - IPAddresses: cfg.AltNames.IPs, - SerialNumber: serial, - NotBefore: caCert.NotBefore, - NotAfter: notAfter, - KeyUsage: keyUsage, - ExtKeyUsage: cfg.Usages, - BasicConstraintsValid: true, - IsCA: isCA, - } - certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &certTmpl, caCert, key.Public(), caKey) - if err != nil { - return nil, err - } - return x509.ParseCertificate(certDERBytes) -} - -// RemoveDuplicateAltNames removes duplicate items in altNames. -func RemoveDuplicateAltNames(altNames *certutil.AltNames) { - if altNames == nil { - return - } - - if altNames.DNSNames != nil { - altNames.DNSNames = sets.NewString(altNames.DNSNames...).List() - } - - ipsKeys := make(map[string]struct{}) - var ips []net.IP - for _, one := range altNames.IPs { - if _, ok := ipsKeys[one.String()]; !ok { - ipsKeys[one.String()] = struct{}{} - ips = append(ips, one) - } - } - altNames.IPs = ips -} - -// CheckCertificatePeriodValidity takes a certificate and prints a warning if its period -// is not valid related to the current time. It does so only if the certificate was not validated already -// by keeping track with a cache. -func CheckCertificatePeriodValidity(baseName string, cert *x509.Certificate) { - certPeriodValidationMutex.Lock() - defer certPeriodValidationMutex.Unlock() - if _, exists := certPeriodValidation[baseName]; exists { - return - } - certPeriodValidation[baseName] = struct{}{} - - if err := ValidateCertPeriod(cert, 0); err != nil { - logrus.Warningf("WARNING: could not validate bounds for certificate %s: %v", baseName, err) - } -} - -// writeCertificateFilesIfNotExist write a new certificate to the given path. -// If there already is a certificate file at the given path; kubeadm tries to load it and check if the values in the -// existing and the expected certificate equals. If they do; kubeadm will just skip writing the file as it's up-to-date, -// otherwise this function returns an error. -func writeCertificateFilesIfNotExist(pkiDir string, baseName string, signingCert *x509.Certificate, cert *x509.Certificate, key crypto.Signer, cfg *CertConfig) error { - - // Checks if the signed certificate exists in the PKI directory - if CertOrKeyExist(pkiDir, baseName) { - // Try to load key from the PKI directory - _, err := TryLoadKeyFromDisk(pkiDir, baseName) - if err != nil { - return errors.Wrapf(err, "failure loading %s key", baseName) - } - - // Try to load certificate from the PKI directory - signedCert, intermediates, err := TryLoadCertChainFromDisk(pkiDir, baseName) - if err != nil { - return errors.Wrapf(err, "failure loading %s certificate", baseName) - } - // Validate period - CheckCertificatePeriodValidity(baseName, signedCert) - - // Check if the existing cert is signed by the given CA - if err := VerifyCertChain(signedCert, intermediates, signingCert); err != nil { - return errors.Errorf("certificate %s is not signed by corresponding CA", baseName) - } - - // Check if the certificate has the correct attributes - if err := validateCertificateWithConfig(signedCert, baseName, cfg); err != nil { - return err - } - - fmt.Printf("[certs] Using the existing %q certificate and key\n", baseName) - } else { - if err := WriteCertAndKey(pkiDir, baseName, cert, key); err != nil { - return errors.Wrapf(err, "failure while saving %s certificate and key", baseName) - } - if HasServerAuth(cert) { - fmt.Printf("[certs] %s serving cert is signed for DNS names %v and IPs %v\n", baseName, cert.DNSNames, cert.IPAddresses) - } - } - - return nil -} diff --git a/cmd/kk/pkg/utils/certs/utils.go b/cmd/kk/pkg/utils/certs/utils.go deleted file mode 100644 index ead7b68ba..000000000 --- a/cmd/kk/pkg/utils/certs/utils.go +++ /dev/null @@ -1,321 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package certs - -import ( - "crypto" - "crypto/ecdsa" - "crypto/elliptic" - cryptorand "crypto/rand" - "crypto/rsa" - "crypto/x509" - "encoding/pem" - "fmt" - "os" - "path/filepath" - "sync" - "time" - - "github.com/pkg/errors" - certutil "k8s.io/client-go/util/cert" - "k8s.io/client-go/util/keyutil" -) - -// CertOrKeyExist returns a boolean whether the cert or the key exists -func CertOrKeyExist(pkiPath, name string) bool { - certificatePath, privateKeyPath := PathsForCertAndKey(pkiPath, name) - - _, certErr := os.Stat(certificatePath) - _, keyErr := os.Stat(privateKeyPath) - if os.IsNotExist(certErr) && os.IsNotExist(keyErr) { - // The cert and the key do not exist - return false - } - - // Both files exist or one of them - return true -} - -// PathsForCertAndKey returns the paths for the certificate and key given the path and basename. -func PathsForCertAndKey(pkiPath, name string) (string, string) { - return pathForCert(pkiPath, name), pathForKey(pkiPath, name) -} - -func pathForCert(pkiPath, name string) string { - return filepath.Join(pkiPath, fmt.Sprintf("%s.pem", name)) -} - -func pathForKey(pkiPath, name string) string { - return filepath.Join(pkiPath, fmt.Sprintf("%s-key.pem", name)) -} - -// TryLoadKeyFromDisk tries to load the key from the disk and validates that it is valid -func TryLoadKeyFromDisk(pkiPath, name string) (crypto.Signer, error) { - privateKeyPath := pathForKey(pkiPath, name) - - // Parse the private key from a file - privKey, err := keyutil.PrivateKeyFromFile(privateKeyPath) - if err != nil { - return nil, errors.Wrapf(err, "couldn't load the private key file %s", privateKeyPath) - } - - // Allow RSA and ECDSA formats only - var key crypto.Signer - switch k := privKey.(type) { - case *rsa.PrivateKey: - key = k - case *ecdsa.PrivateKey: - key = k - default: - return nil, errors.Errorf("the private key file %s is neither in RSA nor ECDSA format", privateKeyPath) - } - - return key, nil -} - -// TryLoadCertChainFromDisk tries to load the cert chain from the disk -func TryLoadCertChainFromDisk(pkiPath, name string) (*x509.Certificate, []*x509.Certificate, error) { - certificatePath := pathForCert(pkiPath, name) - - certs, err := certutil.CertsFromFile(certificatePath) - if err != nil { - return nil, nil, errors.Wrapf(err, "couldn't load the certificate file %s", certificatePath) - } - - cert := certs[0] - intermediates := certs[1:] - - return cert, intermediates, nil -} - -// VerifyCertChain verifies that a certificate has a valid chain of -// intermediate CAs back to the root CA -func VerifyCertChain(cert *x509.Certificate, intermediates []*x509.Certificate, root *x509.Certificate) error { - rootPool := x509.NewCertPool() - rootPool.AddCert(root) - - intermediatePool := x509.NewCertPool() - for _, c := range intermediates { - intermediatePool.AddCert(c) - } - - verifyOptions := x509.VerifyOptions{ - Roots: rootPool, - Intermediates: intermediatePool, - KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, - } - - if _, err := cert.Verify(verifyOptions); err != nil { - return err - } - - return nil -} - -// TryLoadCertAndKeyFromDisk tries to load a cert and a key from the disk and validates that they are valid -func TryLoadCertAndKeyFromDisk(pkiPath, name string) (*x509.Certificate, crypto.Signer, error) { - cert, err := TryLoadCertFromDisk(pkiPath, name) - if err != nil { - return nil, nil, errors.Wrap(err, "failed to load certificate") - } - - key, err := TryLoadKeyFromDisk(pkiPath, name) - if err != nil { - return nil, nil, errors.Wrap(err, "failed to load key") - } - - return cert, key, nil -} - -// TryLoadCertFromDisk tries to load the cert from the disk -func TryLoadCertFromDisk(pkiPath, name string) (*x509.Certificate, error) { - certificatePath := pathForCert(pkiPath, name) - - certs, err := certutil.CertsFromFile(certificatePath) - if err != nil { - return nil, errors.Wrapf(err, "couldn't load the certificate file %s", certificatePath) - } - - // We are only putting one certificate in the certificate pem file, so it's safe to just pick the first one - // TODO: Support multiple certs here in order to be able to rotate certs - cert := certs[0] - - return cert, nil -} - -// CertConfig is a wrapper around certutil.Config extending it with PublicKeyAlgorithm. -type CertConfig struct { - certutil.Config - NotAfter *time.Time - PublicKeyAlgorithm x509.PublicKeyAlgorithm -} - -var ( - // certPeriodValidation is used to store if period validation was done for a certificate - certPeriodValidationMutex sync.Mutex - certPeriodValidation = map[string]struct{}{} -) - -// NewPrivateKey returns a new private key. -var NewPrivateKey = GeneratePrivateKey - -func GeneratePrivateKey(keyType x509.PublicKeyAlgorithm) (crypto.Signer, error) { - if keyType == x509.ECDSA { - return ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader) - } - - return rsa.GenerateKey(cryptorand.Reader, rsaKeySize) -} - -// NewCertificateAuthority creates new certificate and private key for the certificate authority -func NewCertificateAuthority(config *CertConfig) (*x509.Certificate, crypto.Signer, error) { - key, err := NewPrivateKey(config.PublicKeyAlgorithm) - if err != nil { - return nil, nil, errors.Wrap(err, "unable to create private key while generating CA certificate") - } - - cert, err := certutil.NewSelfSignedCACert(config.Config, key) - if err != nil { - return nil, nil, errors.Wrap(err, "unable to create self-signed CA certificate") - } - - return cert, key, nil -} - -// writeCertificateAuthorityFilesIfNotExist write a new certificate Authority to the given path. -// If there already is a certificate file at the given path; kubeadm tries to load it and check if the values in the -// existing and the expected certificate equals. If they do; kubeadm will just skip writing the file as it's up-to-date, -// otherwise this function returns an error. -func writeCertificateAuthorityFilesIfNotExist(pkiDir string, baseName string, caCert *x509.Certificate, caKey crypto.Signer) error { - - // If cert or key exists, we should try to load them - if CertOrKeyExist(pkiDir, baseName) { - - // Try to load .crt and .key from the PKI directory - caCert, _, err := TryLoadCertAndKeyFromDisk(pkiDir, baseName) - if err != nil { - return errors.Wrapf(err, "failure loading %s certificate", baseName) - } - // Validate period - CheckCertificatePeriodValidity(baseName, caCert) - - // Check if the existing cert is a CA - if !caCert.IsCA { - return errors.Errorf("certificate %s is not a CA", baseName) - } - - // kubeadm doesn't validate the existing certificate Authority more than this; - // Basically, if we find a certificate file with the same path; and it is a CA - // kubeadm thinks those files are equal and doesn't bother writing a new file - fmt.Printf("[certs] Using the existing %q certificate and key\n", baseName) - } else { - // Write .crt and .key files to disk - fmt.Printf("[certs] Generating %q certificate and key\n", baseName) - - if err := WriteCertAndKey(pkiDir, baseName, caCert, caKey); err != nil { - return errors.Wrapf(err, "failure while saving %s certificate and key", baseName) - } - } - return nil -} - -// validateCertificateWithConfig makes sure that a given certificate is valid at -// least for the SANs defined in the configuration. -func validateCertificateWithConfig(cert *x509.Certificate, baseName string, cfg *CertConfig) error { - for _, dnsName := range cfg.AltNames.DNSNames { - if err := cert.VerifyHostname(dnsName); err != nil { - return errors.Wrapf(err, "certificate %s is invalid", baseName) - } - } - for _, ipAddress := range cfg.AltNames.IPs { - if err := cert.VerifyHostname(ipAddress.String()); err != nil { - return errors.Wrapf(err, "certificate %s is invalid", baseName) - } - } - return nil -} - -// WriteCertAndKey stores certificate and key at the specified location -func WriteCertAndKey(pkiPath string, name string, cert *x509.Certificate, key crypto.Signer) error { - if err := WriteKey(pkiPath, name, key); err != nil { - return errors.Wrap(err, "couldn't write key") - } - - return WriteCert(pkiPath, name, cert) -} - -// HasServerAuth returns true if the given certificate is a ServerAuth -func HasServerAuth(cert *x509.Certificate) bool { - for i := range cert.ExtKeyUsage { - if cert.ExtKeyUsage[i] == x509.ExtKeyUsageServerAuth { - return true - } - } - return false -} - -// ValidateCertPeriod checks if the certificate is valid relative to the current time -// (+/- offset) -func ValidateCertPeriod(cert *x509.Certificate, offset time.Duration) error { - period := fmt.Sprintf("NotBefore: %v, NotAfter: %v", cert.NotBefore, cert.NotAfter) - now := time.Now().Add(offset) - if now.Before(cert.NotBefore) { - return errors.Errorf("the certificate is not valid yet: %s", period) - } - if now.After(cert.NotAfter) { - return errors.Errorf("the certificate has expired: %s", period) - } - return nil -} - -// WriteKey stores the given key at the given location -func WriteKey(pkiPath, name string, key crypto.Signer) error { - if key == nil { - return errors.New("private key cannot be nil when writing to file") - } - - privateKeyPath := pathForKey(pkiPath, name) - encoded, err := keyutil.MarshalPrivateKeyToPEM(key) - if err != nil { - return errors.Wrapf(err, "unable to marshal private key to PEM") - } - if err := keyutil.WriteKey(privateKeyPath, encoded); err != nil { - return errors.Wrapf(err, "unable to write private key to file %s", privateKeyPath) - } - - return nil -} - -// WriteCert stores the given certificate at the given location -func WriteCert(pkiPath, name string, cert *x509.Certificate) error { - if cert == nil { - return errors.New("certificate cannot be nil when writing to file") - } - - certificatePath := pathForCert(pkiPath, name) - if err := certutil.WriteCert(certificatePath, EncodeCertPEM(cert)); err != nil { - return errors.Wrapf(err, "unable to write certificate to file %s", certificatePath) - } - - return nil -} - -// EncodeCertPEM returns PEM-endcoded certificate data -func EncodeCertPEM(cert *x509.Certificate) []byte { - block := pem.Block{ - Type: CertificateBlockType, - Bytes: cert.Raw, - } - return pem.EncodeToMemory(&block) -} diff --git a/cmd/kk/pkg/utils/client.go b/cmd/kk/pkg/utils/client.go deleted file mode 100644 index 98f22cfa8..000000000 --- a/cmd/kk/pkg/utils/client.go +++ /dev/null @@ -1,58 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package utils - -import ( - "os" - "path/filepath" - - "github.com/pkg/errors" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" -) - -func NewClient(config string) (*kubernetes.Clientset, error) { - var kubeconfig string - if config != "" { - config, err := filepath.Abs(config) - if err != nil { - return nil, errors.Wrap(err, "Failed to look up current directory") - } - kubeconfig = config - } else { - kubeconfig = filepath.Join(homeDir(), ".kube", "config") - } - // use the current context in kubeconfig - configCluster, err := clientcmd.BuildConfigFromFlags("", kubeconfig) - if err != nil { - return nil, err - } - // create the clientset - clientset, err := kubernetes.NewForConfig(configCluster) - if err != nil { - return nil, err - } - - return clientset, nil -} - -func homeDir() string { - if h := os.Getenv("HOME"); h != "" { - return h - } - return os.Getenv("USERPROFILE") -} diff --git a/cmd/kk/pkg/utils/containerruntime/runtime.go b/cmd/kk/pkg/utils/containerruntime/runtime.go deleted file mode 100644 index 46a0dafdd..000000000 --- a/cmd/kk/pkg/utils/containerruntime/runtime.go +++ /dev/null @@ -1,82 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package containerruntime - -import ( - "os" - "strings" - - "github.com/pkg/errors" -) - -const ( - dockerSocket = "/var/run/docker.sock" // The Docker socket is not CRI compatible - - DefaultDockerCRISocket = "/var/run/dockershim.sock" - containerdSocket = "/run/containerd/containerd.sock" -) - -// isExistingSocket checks if path exists and is domain socket -func isExistingSocket(path string) bool { - fileInfo, err := os.Stat(path) - if err != nil { - return false - } - - return fileInfo.Mode()&os.ModeSocket != 0 -} - -// detectCRISocketImpl is separated out only for test purposes, DON'T call it directly, use DetectCRISocket instead -func detectCRISocketImpl(isSocket func(string) bool) (string, error) { - foundCRISockets := []string{} - knownCRISockets := []string{ - // Docker and containerd sockets are special cased below, hence not to be included here - "/var/run/crio/crio.sock", - } - - if isSocket(containerdSocket) { - // Docker 18.09 gets bundled together with containerd, thus having both dockerSocket and containerdSocket present. - // Docker will be deprecated, so prefer to use containerd. - foundCRISockets = append(foundCRISockets, containerdSocket) - } else if isSocket(dockerSocket) { - // the path in dockerSocket is not CRI compatible, hence we should replace it with a CRI compatible socket - foundCRISockets = append(foundCRISockets, DefaultDockerCRISocket) - } - - for _, socket := range knownCRISockets { - if isSocket(socket) { - foundCRISockets = append(foundCRISockets, socket) - } - } - - switch len(foundCRISockets) { - case 0: - // Fall back to Docker if no CRI is detected, we can error out later on if we need it - return containerdSocket, nil - case 1: - // Precisely one CRI found, use that - return foundCRISockets[0], nil - default: - // Multiple CRIs installed? - return "", errors.Errorf("Found multiple CRI sockets, please use --cri-socket to select one: %s", strings.Join(foundCRISockets, ", ")) - } -} - -// DetectCRISocket uses a list of known CRI sockets to detect one. If more than one or none is discovered, an error is returned. -func DetectCRISocket() (string, error) { - return detectCRISocketImpl(isExistingSocket) -} diff --git a/cmd/kk/pkg/utils/util.go b/cmd/kk/pkg/utils/util.go deleted file mode 100644 index e77b48519..000000000 --- a/cmd/kk/pkg/utils/util.go +++ /dev/null @@ -1,57 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package utils - -import ( - "fmt" - "regexp" - "strings" - "text/template" - - "github.com/pkg/errors" - "gopkg.in/yaml.v3" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -var FuncMap = template.FuncMap{"toYaml": ToYAML, "indent": Indent} - -func ResetTmpDir(runtime connector.Runtime) error { - _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "if [ -d %s ]; then rm -rf %s ;fi && mkdir -m 777 -p %s", - common.TmpDir, common.TmpDir, common.TmpDir), false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "reset tmp dir failed") - } - return nil -} - -func ToYAML(v interface{}) string { - data, err := yaml.Marshal(v) - if err != nil { - // Swallow errors inside of a template. - return "" - } - return strings.TrimSuffix(string(data), "\n") -} - -func Indent(n int, text string) string { - startOfLine := regexp.MustCompile(`(?m)^`) - indentation := strings.Repeat(" ", n) - return strings.TrimRight(startOfLine.ReplaceAllLiteralString(text, indentation), " ") -} diff --git a/cmd/kk/pkg/version/kubernetes/version_enum.go b/cmd/kk/pkg/version/kubernetes/version_enum.go deleted file mode 100644 index e0fffdbbf..000000000 --- a/cmd/kk/pkg/version/kubernetes/version_enum.go +++ /dev/null @@ -1,105 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubernetes - -import ( - "fmt" - "sort" - - versionutil "k8s.io/apimachinery/pkg/util/version" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" -) - -type Version int - -const ( - V119 Version = iota - V120 - V121 - V122 - V123 - V124 - V125 - V126 - V127 - V128 -) - -var VersionList = []Version{ - V119, - V120, - V121, - V122, - V123, - V124, - V125, - V126, - V127, - V128, -} - -func (v Version) String() string { - switch v { - case V119: - return "v1.19" - case V120: - return "v1.20" - case V121: - return "v1.21" - case V122: - return "v1.22" - case V123: - return "v1.23" - case V124: - return "v1.24" - case V125: - return "v1.25" - case V126: - return "v1.26" - case V127: - return "v1.27" - case V128: - return "v1.28" - default: - return "invalid option" - } -} - -func VersionSupport(version string) bool { - K8sTargetVersion := versionutil.MustParseSemantic(version) - for i := range VersionList { - if VersionList[i].String() == fmt.Sprintf("v%v.%v", K8sTargetVersion.Major(), K8sTargetVersion.Minor()) { - return true - } - } - return false -} - -// SupportedK8sVersionList returns the supported list of Kubernetes -func SupportedK8sVersionList() []string { - - versionsList := make([]string, 0, len(files.FileSha256["kubeadm"]["amd64"])) - for k := range files.FileSha256["kubeadm"]["amd64"] { - versionsList = append(versionsList, k) - } - sort.Slice(versionsList, func(i, j int) bool { - return versionutil.MustParseSemantic(versionsList[i]).LessThan(versionutil.MustParseSemantic(versionsList[j])) - }) - - return versionsList -} diff --git a/cmd/kk/pkg/version/kubesphere/ks_installer.go b/cmd/kk/pkg/version/kubesphere/ks_installer.go deleted file mode 100644 index 22719db17..000000000 --- a/cmd/kk/pkg/version/kubesphere/ks_installer.go +++ /dev/null @@ -1,260 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubesphere - -import ( - "fmt" - "os" - "text/template" - - versionutil "k8s.io/apimachinery/pkg/util/version" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere/templates" -) - -type KsInstaller struct { - Version string - CRDTemplate *template.Template - ClusterConfigurationTemplate *template.Template - K8sSupportVersions []string - UpgradeSupportVersions []string -} - -func (k *KsInstaller) CCToString() string { - str, err := util.Render(k.ClusterConfigurationTemplate, util.Data{ - "Tag": k.Version, - }) - if err != nil { - os.Exit(0) - } - return str -} - -func (k *KsInstaller) K8sSupport(version string) bool { - k8sVersion := versionutil.MustParseSemantic(version) - for i := range k.K8sSupportVersions { - if k.K8sSupportVersions[i] == fmt.Sprintf("v%v.%v", k8sVersion.Major(), k8sVersion.Minor()) { - return true - } - } - return false -} - -func (k *KsInstaller) UpgradeSupport(version string) bool { - for i := range k.UpgradeSupportVersions { - if k.UpgradeSupportVersions[i] == version { - return true - } - } - return false -} - -var KsV211 = &KsInstaller{ - Version: V211.String(), - CRDTemplate: templates.KsInstaller, - ClusterConfigurationTemplate: templates.V211, - K8sSupportVersions: []string{ - "v1.15", - "v1.16", - "v1.17", - "v1.18", - }, - UpgradeSupportVersions: []string{ - V211.String(), - }, -} - -var KsV300 = &KsInstaller{ - Version: V300.String(), - CRDTemplate: templates.KsInstaller, - ClusterConfigurationTemplate: templates.V300, - K8sSupportVersions: []string{ - "v1.15", - "v1.16", - "v1.17", - "v1.18", - }, - UpgradeSupportVersions: []string{ - V211.String(), - }, -} - -var KsV310 = &KsInstaller{ - Version: V310.String(), - CRDTemplate: templates.KsInstaller, - ClusterConfigurationTemplate: templates.V310, - K8sSupportVersions: []string{ - "v1.15", - "v1.16", - "v1.17", - "v1.18", - "v1.19", - "v1.20", - }, - UpgradeSupportVersions: []string{ - V300.String(), - }, -} - -var KsV311 = &KsInstaller{ - Version: V311.String(), - CRDTemplate: templates.KsInstaller, - ClusterConfigurationTemplate: templates.V311, - K8sSupportVersions: []string{ - "v1.15", - "v1.16", - "v1.17", - "v1.18", - "v1.19", - "v1.20", - }, - UpgradeSupportVersions: []string{ - V300.String(), - V310.String(), - }, -} - -var KsV320 = &KsInstaller{ - Version: V320.String(), - CRDTemplate: templates.KsInstaller, - ClusterConfigurationTemplate: templates.V320, - K8sSupportVersions: []string{ - "v1.19", - "v1.20", - "v1.21", - "v1.22", - }, - UpgradeSupportVersions: []string{ - V310.String(), - V311.String(), - }, -} - -var KsV321 = &KsInstaller{ - Version: V321.String(), - CRDTemplate: templates.KsInstaller, - ClusterConfigurationTemplate: templates.V321, - K8sSupportVersions: []string{ - "v1.19", - "v1.20", - "v1.21", - "v1.22", - }, - UpgradeSupportVersions: []string{ - V310.String(), - V311.String(), - V320.String(), - }, -} - -var KsV330 = &KsInstaller{ - Version: V330.String(), - CRDTemplate: templates.KsInstaller, - ClusterConfigurationTemplate: templates.V330, - K8sSupportVersions: []string{ - "v1.19", - "v1.20", - "v1.21", - "v1.22", - "v1.23", - }, - UpgradeSupportVersions: []string{ - V320.String(), - V321.String(), - }, -} - -var KsV331 = &KsInstaller{ - Version: V331.String(), - CRDTemplate: templates.KsInstaller, - ClusterConfigurationTemplate: templates.V331, - K8sSupportVersions: []string{ - "v1.19", - "v1.20", - "v1.21", - "v1.22", - "v1.23", - "v1.24", - }, - UpgradeSupportVersions: []string{ - V330.String(), - V320.String(), - V321.String(), - }, -} - -var KsV332 = &KsInstaller{ - Version: V332.String(), - CRDTemplate: templates.KsInstaller, - ClusterConfigurationTemplate: templates.V332, - K8sSupportVersions: []string{ - "v1.19", - "v1.20", - "v1.21", - "v1.22", - "v1.23", - "v1.24", - }, - UpgradeSupportVersions: []string{ - V331.String(), - V330.String(), - V320.String(), - V321.String(), - }, -} - -var KsV340 = &KsInstaller{ - Version: V340.String(), - CRDTemplate: templates.KsInstaller, - ClusterConfigurationTemplate: templates.V340, - K8sSupportVersions: []string{ - "v1.19", - "v1.20", - "v1.21", - "v1.22", - "v1.23", - "v1.24", - "v1.25", - "v1.26", - }, - UpgradeSupportVersions: []string{ - V332.String(), - V331.String(), - V330.String(), - }, -} - -var KsV341 = &KsInstaller{ - Version: V341.String(), - CRDTemplate: templates.KsInstaller, - ClusterConfigurationTemplate: templates.V341, - K8sSupportVersions: []string{ - "v1.21", - "v1.22", - "v1.23", - "v1.24", - "v1.25", - "v1.26", - }, - UpgradeSupportVersions: []string{ - V340.String(), - V332.String(), - V331.String(), - V330.String(), - }, -} diff --git a/cmd/kk/pkg/version/kubesphere/templates/cc_v211.go b/cmd/kk/pkg/version/kubesphere/templates/cc_v211.go deleted file mode 100644 index f724df069..000000000 --- a/cmd/kk/pkg/version/kubesphere/templates/cc_v211.go +++ /dev/null @@ -1,92 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var V211 = template.Must(template.New("v2.1.1").Parse( - dedent.Dedent(`--- -apiVersion: v1 -data: - ks-config.yaml: | - --- - local_registry: "" - persistence: - storageClass: "" - etcd: - monitoring: true - endpointIps: localhost - port: 2379 - tlsEnable: true - common: - mysqlVolumeSize: 20Gi - minioVolumeSize: 20Gi - etcdVolumeSize: 20Gi - openldapVolumeSize: 2Gi - redisVolumSize: 2Gi - metrics_server: - enabled: false - console: - enableMultiLogin: False # enable/disable multi login - port: 30880 - monitoring: - prometheusReplicas: 1 - prometheusMemoryRequest: 400Mi - prometheusVolumeSize: 20Gi - grafana: - enabled: false - logging: - enabled: false - elasticsearchMasterReplicas: 1 - elasticsearchDataReplicas: 1 - logsidecarReplicas: 2 - elasticsearchMasterVolumeSize: 4Gi - elasticsearchDataVolumeSize: 20Gi - logMaxAge: 7 - elkPrefix: logstash - containersLogMountedPath: "" - kibana: - enabled: false - openpitrix: - enabled: false - devops: - enabled: false - jenkinsMemoryLim: 2Gi - jenkinsMemoryReq: 1500Mi - jenkinsVolumeSize: 8Gi - jenkinsJavaOpts_Xms: 512m - jenkinsJavaOpts_Xmx: 512m - jenkinsJavaOpts_MaxRAM: 2g - sonarqube: - enabled: false - postgresqlVolumeSize: 8Gi - servicemesh: - enabled: false - notification: - enabled: false - alerting: - enabled: false -kind: ConfigMap -metadata: - name: ks-installer - namespace: kubesphere-system - labels: - version: {{ .Tag }} -`))) diff --git a/cmd/kk/pkg/version/kubesphere/templates/cc_v300.go b/cmd/kk/pkg/version/kubesphere/templates/cc_v300.go deleted file mode 100644 index 91bd178a6..000000000 --- a/cmd/kk/pkg/version/kubesphere/templates/cc_v300.go +++ /dev/null @@ -1,94 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var V300 = template.Must(template.New("v3.0.0").Parse( - dedent.Dedent(`--- -apiVersion: installer.kubesphere.io/v1alpha1 -kind: ClusterConfiguration -metadata: - name: ks-installer - namespace: kubesphere-system - labels: - version: {{ .Tag }} -spec: - zone: "" - local_registry: "" - persistence: - storageClass: "" - authentication: - jwtSecret: "" - etcd: - monitoring: true - endpointIps: localhost - port: 2379 - tlsEnable: true - common: - es: - elasticsearchDataVolumeSize: 20Gi - elasticsearchMasterVolumeSize: 4Gi - elkPrefix: logstash - logMaxAge: 7 - mysqlVolumeSize: 20Gi - minioVolumeSize: 20Gi - etcdVolumeSize: 20Gi - openldapVolumeSize: 2Gi - redisVolumSize: 2Gi - console: - enableMultiLogin: false # enable/disable multi login - port: 30880 - alerting: - enabled: false - auditing: - enabled: false - devops: - enabled: false - jenkinsMemoryLim: 2Gi - jenkinsMemoryReq: 1500Mi - jenkinsVolumeSize: 8Gi - jenkinsJavaOpts_Xms: 512m - jenkinsJavaOpts_Xmx: 512m - jenkinsJavaOpts_MaxRAM: 2g - events: - enabled: false - ruler: - enabled: true - replicas: 2 - logging: - enabled: false - logsidecarReplicas: 2 - metrics_server: - enabled: true - monitoring: - prometheusMemoryRequest: 400Mi - prometheusVolumeSize: 20Gi - multicluster: - clusterRole: none # host | member | none - networkpolicy: - enabled: false - notification: - enabled: false - openpitrix: - enabled: false - servicemesh: - enabled: false -`))) diff --git a/cmd/kk/pkg/version/kubesphere/templates/cc_v310.go b/cmd/kk/pkg/version/kubesphere/templates/cc_v310.go deleted file mode 100644 index e7d14cc92..000000000 --- a/cmd/kk/pkg/version/kubesphere/templates/cc_v310.go +++ /dev/null @@ -1,140 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var V310 = template.Must(template.New("v3.1.0").Parse( - dedent.Dedent(`--- -apiVersion: installer.kubesphere.io/v1alpha1 -kind: ClusterConfiguration -metadata: - name: ks-installer - namespace: kubesphere-system - labels: - version: {{ .Tag }} -spec: - persistence: - storageClass: "" - authentication: - jwtSecret: "" - zone: "" - local_registry: "" - etcd: - monitoring: false - endpointIps: localhost - port: 2379 - tlsEnable: true - common: - redis: - enabled: false - redisVolumSize: 2Gi - openldap: - enabled: false - openldapVolumeSize: 2Gi - minioVolumeSize: 20Gi - monitoring: - endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090 - es: - elasticsearchMasterVolumeSize: 4Gi - elasticsearchDataVolumeSize: 20Gi - logMaxAge: 7 - elkPrefix: logstash - basicAuth: - enabled: false - username: "" - password: "" - externalElasticsearchUrl: "" - externalElasticsearchPort: "" - console: - enableMultiLogin: true - port: 30880 - alerting: - enabled: false - # thanosruler: - # replicas: 1 - # resources: {} - auditing: - enabled: false - devops: - enabled: false - jenkinsMemoryLim: 2Gi - jenkinsMemoryReq: 1500Mi - jenkinsVolumeSize: 8Gi - jenkinsJavaOpts_Xms: 512m - jenkinsJavaOpts_Xmx: 512m - jenkinsJavaOpts_MaxRAM: 2g - events: - enabled: false - ruler: - enabled: true - replicas: 2 - logging: - enabled: false - logsidecar: - enabled: true - replicas: 2 - metrics_server: - enabled: false - monitoring: - storageClass: "" - prometheusMemoryRequest: 400Mi - prometheusVolumeSize: 20Gi - multicluster: - clusterRole: none - network: - networkpolicy: - enabled: false - ippool: - type: none - topology: - type: none - openpitrix: - store: - enabled: false - servicemesh: - enabled: false - kubeedge: - enabled: false - cloudCore: - nodeSelector: {"node-role.kubernetes.io/worker": ""} - tolerations: [] - cloudhubPort: "10000" - cloudhubQuicPort: "10001" - cloudhubHttpsPort: "10002" - cloudstreamPort: "10003" - tunnelPort: "10004" - cloudHub: - advertiseAddress: - - "" - nodeLimit: "100" - service: - cloudhubNodePort: "30000" - cloudhubQuicNodePort: "30001" - cloudhubHttpsNodePort: "30002" - cloudstreamNodePort: "30003" - tunnelNodePort: "30004" - edgeWatcher: - nodeSelector: {"node-role.kubernetes.io/worker": ""} - tolerations: [] - edgeWatcherAgent: - nodeSelector: {"node-role.kubernetes.io/worker": ""} - tolerations: [] -`))) diff --git a/cmd/kk/pkg/version/kubesphere/templates/cc_v311.go b/cmd/kk/pkg/version/kubesphere/templates/cc_v311.go deleted file mode 100644 index dc5ebe31a..000000000 --- a/cmd/kk/pkg/version/kubesphere/templates/cc_v311.go +++ /dev/null @@ -1,140 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var V311 = template.Must(template.New("v3.1.1").Parse( - dedent.Dedent(`--- -apiVersion: installer.kubesphere.io/v1alpha1 -kind: ClusterConfiguration -metadata: - name: ks-installer - namespace: kubesphere-system - labels: - version: {{ .Tag }} -spec: - persistence: - storageClass: "" - authentication: - jwtSecret: "" - zone: "" - local_registry: "" - etcd: - monitoring: false - endpointIps: localhost - port: 2379 - tlsEnable: true - common: - redis: - enabled: false - redisVolumSize: 2Gi - openldap: - enabled: false - openldapVolumeSize: 2Gi - minioVolumeSize: 20Gi - monitoring: - endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090 - es: - elasticsearchMasterVolumeSize: 4Gi - elasticsearchDataVolumeSize: 20Gi - logMaxAge: 7 - elkPrefix: logstash - basicAuth: - enabled: false - username: "" - password: "" - externalElasticsearchUrl: "" - externalElasticsearchPort: "" - console: - enableMultiLogin: true - port: 30880 - alerting: - enabled: false - # thanosruler: - # replicas: 1 - # resources: {} - auditing: - enabled: false - devops: - enabled: false - jenkinsMemoryLim: 2Gi - jenkinsMemoryReq: 1500Mi - jenkinsVolumeSize: 8Gi - jenkinsJavaOpts_Xms: 512m - jenkinsJavaOpts_Xmx: 512m - jenkinsJavaOpts_MaxRAM: 2g - events: - enabled: false - ruler: - enabled: true - replicas: 2 - logging: - enabled: false - logsidecar: - enabled: true - replicas: 2 - metrics_server: - enabled: false - monitoring: - storageClass: "" - prometheusMemoryRequest: 400Mi - prometheusVolumeSize: 20Gi - multicluster: - clusterRole: none - network: - networkpolicy: - enabled: false - ippool: - type: none - topology: - type: none - openpitrix: - store: - enabled: false - servicemesh: - enabled: false - kubeedge: - enabled: false - cloudCore: - nodeSelector: {"node-role.kubernetes.io/worker": ""} - tolerations: [] - cloudhubPort: "10000" - cloudhubQuicPort: "10001" - cloudhubHttpsPort: "10002" - cloudstreamPort: "10003" - tunnelPort: "10004" - cloudHub: - advertiseAddress: - - "" - nodeLimit: "100" - service: - cloudhubNodePort: "30000" - cloudhubQuicNodePort: "30001" - cloudhubHttpsNodePort: "30002" - cloudstreamNodePort: "30003" - tunnelNodePort: "30004" - edgeWatcher: - nodeSelector: {"node-role.kubernetes.io/worker": ""} - tolerations: [] - edgeWatcherAgent: - nodeSelector: {"node-role.kubernetes.io/worker": ""} - tolerations: [] -`))) diff --git a/cmd/kk/pkg/version/kubesphere/templates/cc_v320.go b/cmd/kk/pkg/version/kubesphere/templates/cc_v320.go deleted file mode 100644 index e3720966f..000000000 --- a/cmd/kk/pkg/version/kubesphere/templates/cc_v320.go +++ /dev/null @@ -1,199 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var V320 = template.Must(template.New("v3.2.0").Parse( - dedent.Dedent(` ---- -apiVersion: installer.kubesphere.io/v1alpha1 -kind: ClusterConfiguration -metadata: - name: ks-installer - namespace: kubesphere-system - labels: - version: {{ .Tag }} -spec: - persistence: - storageClass: "" - authentication: - jwtSecret: "" - zone: "" - local_registry: "" - # dev_tag: "" - etcd: - monitoring: false - endpointIps: localhost - port: 2379 - tlsEnable: true - common: - core: - console: - enableMultiLogin: true - port: 30880 - type: NodePort - # apiserver: - # resources: {} - # controllerManager: - # resources: {} - redis: - enabled: false - volumeSize: 2Gi - openldap: - enabled: false - volumeSize: 2Gi - minio: - volumeSize: 20Gi - monitoring: - # type: external - endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090 - GPUMonitoring: - enabled: false - gpu: - kinds: - - resourceName: "nvidia.com/gpu" - resourceType: "GPU" - default: true - es: - # master: - # volumeSize: 4Gi - # replicas: 1 - # resources: {} - # data: - # volumeSize: 20Gi - # replicas: 1 - # resources: {} - logMaxAge: 7 - elkPrefix: logstash - basicAuth: - enabled: false - username: "" - password: "" - externalElasticsearchUrl: "" - externalElasticsearchPort: "" - alerting: - enabled: false - # thanosruler: - # replicas: 1 - # resources: {} - auditing: - enabled: false - # operator: - # resources: {} - # webhook: - # resources: {} - devops: - enabled: false - jenkinsMemoryLim: 2Gi - jenkinsMemoryReq: 1500Mi - jenkinsVolumeSize: 8Gi - jenkinsJavaOpts_Xms: 512m - jenkinsJavaOpts_Xmx: 512m - jenkinsJavaOpts_MaxRAM: 2g - events: - enabled: false - # operator: - # resources: {} - # exporter: - # resources: {} - # ruler: - # enabled: true - # replicas: 2 - # resources: {} - logging: - enabled: false - containerruntime: docker - logsidecar: - enabled: true - replicas: 2 - # resources: {} - metrics_server: - enabled: false - monitoring: - storageClass: "" - # kube_rbac_proxy: - # resources: {} - # kube_state_metrics: - # resources: {} - # prometheus: - # replicas: 1 - # volumeSize: 20Gi - # resources: {} - # operator: - # resources: {} - # adapter: - # resources: {} - # node_exporter: - # resources: {} - # alertmanager: - # replicas: 1 - # resources: {} - # notification_manager: - # resources: {} - # operator: - # resources: {} - # proxy: - # resources: {} - gpu: - nvidia_dcgm_exporter: - enabled: false - # resources: {} - multicluster: - clusterRole: none - network: - networkpolicy: - enabled: false - ippool: - type: none - topology: - type: none - openpitrix: - store: - enabled: false - servicemesh: - enabled: false - kubeedge: - enabled: false - cloudCore: - nodeSelector: {"node-role.kubernetes.io/worker": ""} - tolerations: [] - cloudhubPort: "10000" - cloudhubQuicPort: "10001" - cloudhubHttpsPort: "10002" - cloudstreamPort: "10003" - tunnelPort: "10004" - cloudHub: - advertiseAddress: - - "" - nodeLimit: "100" - service: - cloudhubNodePort: "30000" - cloudhubQuicNodePort: "30001" - cloudhubHttpsNodePort: "30002" - cloudstreamNodePort: "30003" - tunnelNodePort: "30004" - edgeWatcher: - nodeSelector: {"node-role.kubernetes.io/worker": ""} - tolerations: [] - edgeWatcherAgent: - nodeSelector: {"node-role.kubernetes.io/worker": ""} - tolerations: [] -`))) diff --git a/cmd/kk/pkg/version/kubesphere/templates/cc_v321.go b/cmd/kk/pkg/version/kubesphere/templates/cc_v321.go deleted file mode 100644 index 5fc4f35d7..000000000 --- a/cmd/kk/pkg/version/kubesphere/templates/cc_v321.go +++ /dev/null @@ -1,200 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var V321 = template.Must(template.New("v3.2.1").Parse( - dedent.Dedent(` ---- -apiVersion: installer.kubesphere.io/v1alpha1 -kind: ClusterConfiguration -metadata: - name: ks-installer - namespace: kubesphere-system - labels: - version: {{ .Tag }} -spec: - persistence: - storageClass: "" - authentication: - jwtSecret: "" - zone: "" - local_registry: "" - namespace_override: "" - # dev_tag: "" - etcd: - monitoring: true - endpointIps: localhost - port: 2379 - tlsEnable: true - common: - core: - console: - enableMultiLogin: true - port: 30880 - type: NodePort - # apiserver: - # resources: {} - # controllerManager: - # resources: {} - redis: - enabled: false - volumeSize: 2Gi - openldap: - enabled: false - volumeSize: 2Gi - minio: - volumeSize: 20Gi - monitoring: - # type: external - endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090 - GPUMonitoring: - enabled: false - gpu: - kinds: - - resourceName: "nvidia.com/gpu" - resourceType: "GPU" - default: true - es: - # master: - # volumeSize: 4Gi - # replicas: 1 - # resources: {} - # data: - # volumeSize: 20Gi - # replicas: 1 - # resources: {} - logMaxAge: 7 - elkPrefix: logstash - basicAuth: - enabled: false - username: "" - password: "" - externalElasticsearchHost: "" - externalElasticsearchPort: "" - alerting: - enabled: false - # thanosruler: - # replicas: 1 - # resources: {} - auditing: - enabled: false - # operator: - # resources: {} - # webhook: - # resources: {} - devops: - enabled: false - jenkinsMemoryLim: 2Gi - jenkinsMemoryReq: 1500Mi - jenkinsVolumeSize: 8Gi - jenkinsJavaOpts_Xms: 512m - jenkinsJavaOpts_Xmx: 512m - jenkinsJavaOpts_MaxRAM: 2g - events: - enabled: false - # operator: - # resources: {} - # exporter: - # resources: {} - # ruler: - # enabled: true - # replicas: 2 - # resources: {} - logging: - enabled: false - containerruntime: docker - logsidecar: - enabled: true - replicas: 2 - # resources: {} - metrics_server: - enabled: false - monitoring: - storageClass: "" - # kube_rbac_proxy: - # resources: {} - # kube_state_metrics: - # resources: {} - # prometheus: - # replicas: 1 - # volumeSize: 20Gi - # resources: {} - # operator: - # resources: {} - # adapter: - # resources: {} - # node_exporter: - # resources: {} - # alertmanager: - # replicas: 1 - # resources: {} - # notification_manager: - # resources: {} - # operator: - # resources: {} - # proxy: - # resources: {} - gpu: - nvidia_dcgm_exporter: - enabled: false - # resources: {} - multicluster: - clusterRole: none - network: - networkpolicy: - enabled: false - ippool: - type: none - topology: - type: none - openpitrix: - store: - enabled: false - servicemesh: - enabled: false - kubeedge: - enabled: false - cloudCore: - nodeSelector: {"node-role.kubernetes.io/worker": ""} - tolerations: [] - cloudhubPort: "10000" - cloudhubQuicPort: "10001" - cloudhubHttpsPort: "10002" - cloudstreamPort: "10003" - tunnelPort: "10004" - cloudHub: - advertiseAddress: - - "" - nodeLimit: "100" - service: - cloudhubNodePort: "30000" - cloudhubQuicNodePort: "30001" - cloudhubHttpsNodePort: "30002" - cloudstreamNodePort: "30003" - tunnelNodePort: "30004" - edgeWatcher: - nodeSelector: {"node-role.kubernetes.io/worker": ""} - tolerations: [] - edgeWatcherAgent: - nodeSelector: {"node-role.kubernetes.io/worker": ""} - tolerations: [] -`))) diff --git a/cmd/kk/pkg/version/kubesphere/templates/cc_v330.go b/cmd/kk/pkg/version/kubesphere/templates/cc_v330.go deleted file mode 100644 index a55614170..000000000 --- a/cmd/kk/pkg/version/kubesphere/templates/cc_v330.go +++ /dev/null @@ -1,204 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var V330 = template.Must(template.New("v3.3.0").Parse( - dedent.Dedent(` ---- -apiVersion: installer.kubesphere.io/v1alpha1 -kind: ClusterConfiguration -metadata: - name: ks-installer - namespace: kubesphere-system - labels: - version: {{ .Tag }} -spec: - persistence: - storageClass: "" - authentication: - jwtSecret: "" - zone: "" - local_registry: "" - namespace_override: "" - # dev_tag: "" - etcd: - monitoring: false - endpointIps: localhost - port: 2379 - tlsEnable: true - common: - core: - console: - enableMultiLogin: true - port: 30880 - type: NodePort - # apiserver: - # resources: {} - # controllerManager: - # resources: {} - redis: - enabled: false - volumeSize: 2Gi - openldap: - enabled: false - volumeSize: 2Gi - minio: - volumeSize: 20Gi - monitoring: - # type: external - endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090 - GPUMonitoring: - enabled: false - gpu: - kinds: - - resourceName: "nvidia.com/gpu" - resourceType: "GPU" - default: true - es: - # master: - # volumeSize: 4Gi - # replicas: 1 - # resources: {} - # data: - # volumeSize: 20Gi - # replicas: 1 - # resources: {} - logMaxAge: 7 - elkPrefix: logstash - basicAuth: - enabled: false - username: "" - password: "" - externalElasticsearchHost: "" - externalElasticsearchPort: "" - alerting: - enabled: false - # thanosruler: - # replicas: 1 - # resources: {} - auditing: - enabled: false - # operator: - # resources: {} - # webhook: - # resources: {} - devops: - enabled: false - # resources: {} - jenkinsMemoryLim: 2Gi - jenkinsMemoryReq: 1500Mi - jenkinsVolumeSize: 8Gi - jenkinsJavaOpts_Xms: 1200m - jenkinsJavaOpts_Xmx: 1600m - jenkinsJavaOpts_MaxRAM: 2g - events: - enabled: false - # operator: - # resources: {} - # exporter: - # resources: {} - # ruler: - # enabled: true - # replicas: 2 - # resources: {} - logging: - enabled: false - logsidecar: - enabled: true - replicas: 2 - # resources: {} - metrics_server: - enabled: false - monitoring: - storageClass: "" - node_exporter: - port: 9100 - # resources: {} - # kube_rbac_proxy: - # resources: {} - # kube_state_metrics: - # resources: {} - # prometheus: - # replicas: 1 - # volumeSize: 20Gi - # resources: {} - # operator: - # resources: {} - # alertmanager: - # replicas: 1 - # resources: {} - # notification_manager: - # resources: {} - # operator: - # resources: {} - # proxy: - # resources: {} - gpu: - nvidia_dcgm_exporter: - enabled: false - # resources: {} - multicluster: - clusterRole: none - network: - networkpolicy: - enabled: false - ippool: - type: none - topology: - type: none - openpitrix: - store: - enabled: false - servicemesh: - enabled: false - istio: - components: - ingressGateways: - - name: istio-ingressgateway - enabled: false - cni: - enabled: false - edgeruntime: - enabled: false - kubeedge: - enabled: false - cloudCore: - cloudHub: - advertiseAddress: - - "" - service: - cloudhubNodePort: "30000" - cloudhubQuicNodePort: "30001" - cloudhubHttpsNodePort: "30002" - cloudstreamNodePort: "30003" - tunnelNodePort: "30004" - # resources: {} - # hostNetWork: false - iptables-manager: - enabled: true - mode: "external" - # resources: {} - # edgeService: - # resources: {} - terminal: - timeout: 600 -`))) diff --git a/cmd/kk/pkg/version/kubesphere/templates/cc_v331.go b/cmd/kk/pkg/version/kubesphere/templates/cc_v331.go deleted file mode 100644 index 1b1b0e5bb..000000000 --- a/cmd/kk/pkg/version/kubesphere/templates/cc_v331.go +++ /dev/null @@ -1,202 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -var V331 = template.Must(template.New("v3.3.1").Parse( - dedent.Dedent(` ---- -apiVersion: installer.kubesphere.io/v1alpha1 -kind: ClusterConfiguration -metadata: - name: ks-installer - namespace: kubesphere-system - labels: - version: {{ .Tag }} -spec: - persistence: - storageClass: "" - authentication: - jwtSecret: "" - zone: "" - local_registry: "" - namespace_override: "" - # dev_tag: "" - etcd: - monitoring: false - endpointIps: localhost - port: 2379 - tlsEnable: true - common: - core: - console: - enableMultiLogin: true - port: 30880 - type: NodePort - # apiserver: - # resources: {} - # controllerManager: - # resources: {} - redis: - enabled: false - volumeSize: 2Gi - openldap: - enabled: false - volumeSize: 2Gi - minio: - volumeSize: 20Gi - monitoring: - # type: external - endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090 - GPUMonitoring: - enabled: false - gpu: - kinds: - - resourceName: "nvidia.com/gpu" - resourceType: "GPU" - default: true - es: - # master: - # volumeSize: 4Gi - # replicas: 1 - # resources: {} - # data: - # volumeSize: 20Gi - # replicas: 1 - # resources: {} - logMaxAge: 7 - elkPrefix: logstash - basicAuth: - enabled: false - username: "" - password: "" - externalElasticsearchHost: "" - externalElasticsearchPort: "" - alerting: - enabled: false - # thanosruler: - # replicas: 1 - # resources: {} - auditing: - enabled: false - # operator: - # resources: {} - # webhook: - # resources: {} - devops: - enabled: false - # resources: {} - jenkinsMemoryLim: 8Gi - jenkinsMemoryReq: 4Gi - jenkinsVolumeSize: 8Gi - events: - enabled: false - # operator: - # resources: {} - # exporter: - # resources: {} - # ruler: - # enabled: true - # replicas: 2 - # resources: {} - logging: - enabled: false - logsidecar: - enabled: true - replicas: 2 - # resources: {} - metrics_server: - enabled: false - monitoring: - storageClass: "" - node_exporter: - port: 9100 - # resources: {} - # kube_rbac_proxy: - # resources: {} - # kube_state_metrics: - # resources: {} - # prometheus: - # replicas: 1 - # volumeSize: 20Gi - # resources: {} - # operator: - # resources: {} - # alertmanager: - # replicas: 1 - # resources: {} - # notification_manager: - # resources: {} - # operator: - # resources: {} - # proxy: - # resources: {} - gpu: - nvidia_dcgm_exporter: - enabled: false - # resources: {} - multicluster: - clusterRole: none - network: - networkpolicy: - enabled: false - ippool: - type: none - topology: - type: none - openpitrix: - store: - enabled: false - servicemesh: - enabled: false - istio: - components: - ingressGateways: - - name: istio-ingressgateway - enabled: false - cni: - enabled: false - edgeruntime: - enabled: false - kubeedge: - enabled: false - cloudCore: - cloudHub: - advertiseAddress: - - "" - service: - cloudhubNodePort: "30000" - cloudhubQuicNodePort: "30001" - cloudhubHttpsNodePort: "30002" - cloudstreamNodePort: "30003" - tunnelNodePort: "30004" - # resources: {} - # hostNetWork: false - iptables-manager: - enabled: true - mode: "external" - # resources: {} - # edgeService: - # resources: {} - terminal: - timeout: 600 -`))) diff --git a/cmd/kk/pkg/version/kubesphere/templates/cc_v332.go b/cmd/kk/pkg/version/kubesphere/templates/cc_v332.go deleted file mode 100644 index 7463d1c53..000000000 --- a/cmd/kk/pkg/version/kubesphere/templates/cc_v332.go +++ /dev/null @@ -1,202 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -var V332 = template.Must(template.New("v3.3.2").Parse( - dedent.Dedent(` ---- -apiVersion: installer.kubesphere.io/v1alpha1 -kind: ClusterConfiguration -metadata: - name: ks-installer - namespace: kubesphere-system - labels: - version: {{ .Tag }} -spec: - persistence: - storageClass: "" - authentication: - jwtSecret: "" - zone: "" - local_registry: "" - namespace_override: "" - # dev_tag: "" - etcd: - monitoring: false - endpointIps: localhost - port: 2379 - tlsEnable: true - common: - core: - console: - enableMultiLogin: true - port: 30880 - type: NodePort - # apiserver: - # resources: {} - # controllerManager: - # resources: {} - redis: - enabled: false - volumeSize: 2Gi - openldap: - enabled: false - volumeSize: 2Gi - minio: - volumeSize: 20Gi - monitoring: - # type: external - endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090 - GPUMonitoring: - enabled: false - gpu: - kinds: - - resourceName: "nvidia.com/gpu" - resourceType: "GPU" - default: true - es: - # master: - # volumeSize: 4Gi - # replicas: 1 - # resources: {} - # data: - # volumeSize: 20Gi - # replicas: 1 - # resources: {} - logMaxAge: 7 - elkPrefix: logstash - basicAuth: - enabled: false - username: "" - password: "" - externalElasticsearchHost: "" - externalElasticsearchPort: "" - alerting: - enabled: false - # thanosruler: - # replicas: 1 - # resources: {} - auditing: - enabled: false - # operator: - # resources: {} - # webhook: - # resources: {} - devops: - enabled: false - # resources: {} - jenkinsMemoryLim: 8Gi - jenkinsMemoryReq: 4Gi - jenkinsVolumeSize: 8Gi - events: - enabled: false - # operator: - # resources: {} - # exporter: - # resources: {} - # ruler: - # enabled: true - # replicas: 2 - # resources: {} - logging: - enabled: false - logsidecar: - enabled: true - replicas: 2 - # resources: {} - metrics_server: - enabled: false - monitoring: - storageClass: "" - node_exporter: - port: 9100 - # resources: {} - # kube_rbac_proxy: - # resources: {} - # kube_state_metrics: - # resources: {} - # prometheus: - # replicas: 1 - # volumeSize: 20Gi - # resources: {} - # operator: - # resources: {} - # alertmanager: - # replicas: 1 - # resources: {} - # notification_manager: - # resources: {} - # operator: - # resources: {} - # proxy: - # resources: {} - gpu: - nvidia_dcgm_exporter: - enabled: false - # resources: {} - multicluster: - clusterRole: none - network: - networkpolicy: - enabled: false - ippool: - type: none - topology: - type: none - openpitrix: - store: - enabled: false - servicemesh: - enabled: false - istio: - components: - ingressGateways: - - name: istio-ingressgateway - enabled: false - cni: - enabled: false - edgeruntime: - enabled: false - kubeedge: - enabled: false - cloudCore: - cloudHub: - advertiseAddress: - - "" - service: - cloudhubNodePort: "30000" - cloudhubQuicNodePort: "30001" - cloudhubHttpsNodePort: "30002" - cloudstreamNodePort: "30003" - tunnelNodePort: "30004" - # resources: {} - # hostNetWork: false - iptables-manager: - enabled: true - mode: "external" - # resources: {} - # edgeService: - # resources: {} - terminal: - timeout: 600 -`))) diff --git a/cmd/kk/pkg/version/kubesphere/templates/cc_v340.go b/cmd/kk/pkg/version/kubesphere/templates/cc_v340.go deleted file mode 100644 index b76fe1fc7..000000000 --- a/cmd/kk/pkg/version/kubesphere/templates/cc_v340.go +++ /dev/null @@ -1,230 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -var V340 = template.Must(template.New("v3.4.0").Parse( - dedent.Dedent(` ---- -apiVersion: installer.kubesphere.io/v1alpha1 -kind: ClusterConfiguration -metadata: - name: ks-installer - namespace: kubesphere-system - labels: - version: {{ .Tag }} -spec: - persistence: - storageClass: "" - authentication: - jwtSecret: "" - zone: "" - local_registry: "" - namespace_override: "" - # dev_tag: "" - etcd: - monitoring: false - endpointIps: localhost - port: 2379 - tlsEnable: true - common: - core: - console: - enableMultiLogin: true - port: 30880 - type: NodePort - # apiserver: - # resources: {} - # controllerManager: - # resources: {} - redis: - enabled: false - enableHA: false - volumeSize: 2Gi - openldap: - enabled: false - volumeSize: 2Gi - minio: - volumeSize: 20Gi - monitoring: - # type: external - endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090 - GPUMonitoring: - enabled: false - gpu: - kinds: - - resourceName: "nvidia.com/gpu" - resourceType: "GPU" - default: true - es: - # master: - # volumeSize: 4Gi - # replicas: 1 - # resources: {} - # data: - # volumeSize: 20Gi - # replicas: 1 - # resources: {} - logMaxAge: 7 - elkPrefix: logstash - basicAuth: - enabled: false - username: "" - password: "" - externalElasticsearchHost: "" - externalElasticsearchPort: "" - opensearch: - # master: - # volumeSize: 4Gi - # replicas: 1 - # resources: {} - # data: - # volumeSize: 20Gi - # replicas: 1 - # resources: {} - enabled: true - logMaxAge: 7 - opensearchPrefix: whizard - basicAuth: - enabled: true - username: "admin" - password: "admin" - externalOpensearchHost: "" - externalOpensearchPort: "" - dashboard: - enabled: false - alerting: - enabled: false - # thanosruler: - # replicas: 1 - # resources: {} - auditing: - enabled: false - # operator: - # resources: {} - # webhook: - # resources: {} - devops: - enabled: false - jenkinsCpuReq: 0.5 - jenkinsCpuLim: 1 - jenkinsMemoryReq: 4Gi - jenkinsMemoryLim: 4Gi - jenkinsVolumeSize: 16Gi - events: - enabled: false - # operator: - # resources: {} - # exporter: - # resources: {} - # ruler: - # enabled: true - # replicas: 2 - # resources: {} - logging: - enabled: false - logsidecar: - enabled: true - replicas: 2 - # resources: {} - metrics_server: - enabled: false - monitoring: - storageClass: "" - node_exporter: - port: 9100 - # resources: {} - # kube_rbac_proxy: - # resources: {} - # kube_state_metrics: - # resources: {} - # prometheus: - # replicas: 1 - # volumeSize: 20Gi - # resources: {} - # operator: - # resources: {} - # alertmanager: - # replicas: 1 - # resources: {} - # notification_manager: - # resources: {} - # operator: - # resources: {} - # proxy: - # resources: {} - gpu: - nvidia_dcgm_exporter: - enabled: false - # resources: {} - multicluster: - clusterRole: none - network: - networkpolicy: - enabled: false - ippool: - type: none - topology: - type: none - openpitrix: - store: - enabled: false - servicemesh: - enabled: false - istio: - components: - ingressGateways: - - name: istio-ingressgateway - enabled: false - cni: - enabled: false - edgeruntime: - enabled: false - kubeedge: - enabled: false - cloudCore: - cloudHub: - advertiseAddress: - - "" - service: - cloudhubNodePort: "30000" - cloudhubQuicNodePort: "30001" - cloudhubHttpsNodePort: "30002" - cloudstreamNodePort: "30003" - tunnelNodePort: "30004" - # resources: {} - # hostNetWork: false - iptables-manager: - enabled: true - mode: "external" - # resources: {} - # edgeService: - # resources: {} - gatekeeper: - enabled: false - # controller_manager: - # resources: {} - # audit: - # resources: {} - terminal: - timeout: 600 -`))) diff --git a/cmd/kk/pkg/version/kubesphere/templates/cc_v341.go b/cmd/kk/pkg/version/kubesphere/templates/cc_v341.go deleted file mode 100644 index fd2ec6c02..000000000 --- a/cmd/kk/pkg/version/kubesphere/templates/cc_v341.go +++ /dev/null @@ -1,229 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -var V341 = template.Must(template.New("v3.4.1").Parse( - dedent.Dedent(` ---- -apiVersion: installer.kubesphere.io/v1alpha1 -kind: ClusterConfiguration -metadata: - name: ks-installer - namespace: kubesphere-system - labels: - version: {{ .Tag }} -spec: - persistence: - storageClass: "" - authentication: - jwtSecret: "" - local_registry: "" - # dev_tag: "" - etcd: - monitoring: false - endpointIps: localhost - port: 2379 - tlsEnable: true - common: - core: - console: - enableMultiLogin: true - port: 30880 - type: NodePort - # apiserver: - # resources: {} - # controllerManager: - # resources: {} - redis: - enabled: false - enableHA: false - volumeSize: 2Gi - openldap: - enabled: false - volumeSize: 2Gi - minio: - volumeSize: 20Gi - monitoring: - # type: external - endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090 - GPUMonitoring: - enabled: false - gpu: - kinds: - - resourceName: "nvidia.com/gpu" - resourceType: "GPU" - default: true - es: - # master: - # volumeSize: 4Gi - # replicas: 1 - # resources: {} - # data: - # volumeSize: 20Gi - # replicas: 1 - # resources: {} - enabled: false - logMaxAge: 7 - elkPrefix: logstash - basicAuth: - enabled: false - username: "" - password: "" - externalElasticsearchHost: "" - externalElasticsearchPort: "" - opensearch: - # master: - # volumeSize: 4Gi - # replicas: 1 - # resources: {} - # data: - # volumeSize: 20Gi - # replicas: 1 - # resources: {} - enabled: true - logMaxAge: 7 - opensearchPrefix: whizard - basicAuth: - enabled: true - username: "admin" - password: "admin" - externalOpensearchHost: "" - externalOpensearchPort: "" - dashboard: - enabled: false - alerting: - enabled: false - # thanosruler: - # replicas: 1 - # resources: {} - auditing: - enabled: false - # operator: - # resources: {} - # webhook: - # resources: {} - devops: - enabled: false - jenkinsCpuReq: 0.5 - jenkinsCpuLim: 1 - jenkinsMemoryReq: 4Gi - jenkinsMemoryLim: 4Gi - jenkinsVolumeSize: 16Gi - events: - enabled: false - # operator: - # resources: {} - # exporter: - # resources: {} - ruler: - enabled: true - replicas: 2 - # resources: {} - logging: - enabled: false - logsidecar: - enabled: true - replicas: 2 - # resources: {} - metrics_server: - enabled: false - monitoring: - storageClass: "" - node_exporter: - port: 9100 - # resources: {} - # kube_rbac_proxy: - # resources: {} - # kube_state_metrics: - # resources: {} - # prometheus: - # replicas: 1 - # volumeSize: 20Gi - # resources: {} - # operator: - # resources: {} - # alertmanager: - # replicas: 1 - # resources: {} - # notification_manager: - # resources: {} - # operator: - # resources: {} - # proxy: - # resources: {} - gpu: - nvidia_dcgm_exporter: - enabled: false - # resources: {} - multicluster: - clusterRole: none - network: - networkpolicy: - enabled: false - ippool: - type: none - topology: - type: none - openpitrix: - store: - enabled: false - servicemesh: - enabled: false - istio: - components: - ingressGateways: - - name: istio-ingressgateway - enabled: false - cni: - enabled: false - edgeruntime: - enabled: false - kubeedge: - enabled: false - cloudCore: - cloudHub: - advertiseAddress: - - "" - service: - cloudhubNodePort: "30000" - cloudhubQuicNodePort: "30001" - cloudhubHttpsNodePort: "30002" - cloudstreamNodePort: "30003" - tunnelNodePort: "30004" - # resources: {} - # hostNetWork: false - iptables-manager: - enabled: true - mode: "external" - # resources: {} - # edgeService: - # resources: {} - gatekeeper: - enabled: false - # controller_manager: - # resources: {} - # audit: - # resources: {} - terminal: - timeout: 600 -`))) diff --git a/cmd/kk/pkg/version/kubesphere/templates/installer.go b/cmd/kk/pkg/version/kubesphere/templates/installer.go deleted file mode 100644 index 943b3ae63..000000000 --- a/cmd/kk/pkg/version/kubesphere/templates/installer.go +++ /dev/null @@ -1,336 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -var ( - KsInstaller = template.Must(template.New("kubesphere.yaml").Parse( - dedent.Dedent(` ---- -apiVersion: v1 -kind: Namespace -metadata: - name: kubesphere-system - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: ks-installer - namespace: kubesphere-system - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: clusterconfigurations.installer.kubesphere.io -spec: - group: installer.kubesphere.io - versions: - - name: v1alpha1 - served: true - storage: true - schema: - openAPIV3Schema: - type: object - properties: - spec: - type: object - x-kubernetes-preserve-unknown-fields: true - status: - type: object - x-kubernetes-preserve-unknown-fields: true - scope: Namespaced - names: - plural: clusterconfigurations - singular: clusterconfiguration - kind: ClusterConfiguration - shortNames: - - cc - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: ks-installer -rules: -- apiGroups: - - "" - resources: - - '*' - verbs: - - '*' -- apiGroups: - - apps - resources: - - '*' - verbs: - - '*' -- apiGroups: - - extensions - resources: - - '*' - verbs: - - '*' -- apiGroups: - - batch - resources: - - '*' - verbs: - - '*' -- apiGroups: - - rbac.authorization.k8s.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - apiregistration.k8s.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - apiextensions.k8s.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - tenant.kubesphere.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - certificates.k8s.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - devops.kubesphere.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - monitoring.coreos.com - resources: - - '*' - verbs: - - '*' -- apiGroups: - - logging.kubesphere.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - jaegertracing.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - storage.k8s.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - admissionregistration.k8s.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - policy - resources: - - '*' - verbs: - - '*' -- apiGroups: - - autoscaling - resources: - - '*' - verbs: - - '*' -- apiGroups: - - networking.istio.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - config.istio.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - iam.kubesphere.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - notification.kubesphere.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - auditing.kubesphere.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - events.kubesphere.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - core.kubefed.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - installer.kubesphere.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - storage.kubesphere.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - security.istio.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - monitoring.kiali.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - kiali.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - networking.k8s.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - kubeedge.kubesphere.io - - edgeruntime.kubesphere.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - types.kubefed.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - monitoring.kubesphere.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - application.kubesphere.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - alerting.kubesphere.io - resources: - - '*' - verbs: - - '*' - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: ks-installer -subjects: -- kind: ServiceAccount - name: ks-installer - namespace: kubesphere-system -roleRef: - kind: ClusterRole - name: ks-installer - apiGroup: rbac.authorization.k8s.io - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ks-installer - namespace: kubesphere-system - labels: - app: ks-installer - version: {{ .Tag }} -spec: - replicas: 1 - selector: - matchLabels: - app: ks-installer - template: - metadata: - labels: - app: ks-installer - spec: - serviceAccountName: ks-installer - containers: - - name: installer - image: {{ .Repo }}/ks-installer:{{ .Tag }} - imagePullPolicy: IfNotPresent - volumeMounts: - - mountPath: /etc/localtime - name: host-time - volumes: - - hostPath: - path: /etc/localtime - type: "" - name: host-time - - `))) -) diff --git a/cmd/kk/pkg/version/kubesphere/version_enum.go b/cmd/kk/pkg/version/kubesphere/version_enum.go deleted file mode 100644 index d29c71a00..000000000 --- a/cmd/kk/pkg/version/kubesphere/version_enum.go +++ /dev/null @@ -1,176 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubesphere - -import ( - "fmt" - "strings" - - versionutil "k8s.io/apimachinery/pkg/util/version" -) - -type Version int - -const ( - V211 Version = iota - V300 - V310 - V311 - V320 - V321 - V330 - V331 - V332 - V340 - V341 -) - -var VersionList = []Version{ - V211, - V300, - V310, - V311, - V320, - V321, - V330, - V331, - V332, - V340, - V341, -} - -var VersionMap = map[string]*KsInstaller{ - V211.String(): KsV211, - V300.String(): KsV300, - V310.String(): KsV310, - V311.String(): KsV311, - V320.String(): KsV320, - V321.String(): KsV321, - V330.String(): KsV330, - V331.String(): KsV331, - V332.String(): KsV332, - V340.String(): KsV340, - V341.String(): KsV341, -} - -var CNSource = map[string]bool{ - V310.String(): true, - V311.String(): true, - V320.String(): true, - V321.String(): true, - V330.String(): true, - V331.String(): true, - V332.String(): true, - V340.String(): true, - V341.String(): true, -} - -func (v Version) String() string { - switch v { - case V211: - return "v2.1.1" - case V300: - return "v3.0.0" - case V310: - return "v3.1.0" - case V311: - return "v3.1.1" - case V320: - return "v3.2.0" - case V321: - return "v3.2.1" - case V330: - return "v3.3.0" - case V331: - return "v3.3.1" - case V332: - return "v3.3.2" - case V340: - return "v3.4.0" - case V341: - return "v3.4.1" - default: - return "invalid option" - } -} - -func VersionsStringArr() []string { - strArr := make([]string, 0, len(VersionList)) - for i, v := range VersionList { - strArr[i] = v.String() - } - return strArr -} - -func StabledVersionSupport(version string) (*KsInstaller, bool) { - if ks, ok := VersionMap[version]; ok { - return ks, true - } - return nil, false -} - -func LatestRelease(version string) (*KsInstaller, bool) { - if strings.HasPrefix(version, "nightly-") || - version == "latest" || - version == "master" || - strings.Contains(version, "release") { - return Latest(), true - } - - v, err := versionutil.ParseGeneric(version) - if err != nil { - return nil, false - } - - str := fmt.Sprintf("v%s", v.String()) - if ks, ok := StabledVersionSupport(str); ok { - if ks.Version == Latest().Version { - return ks, true - } - return nil, false - } - - return nil, false -} - -func DevRelease(version string) (*KsInstaller, bool) { - if strings.HasPrefix(version, "nightly-") || - version == "latest" || - version == "master" || - strings.Contains(version, "release") { - return Latest(), true - } - - if _, ok := StabledVersionSupport(version); ok { - return nil, false - } - - v, err := versionutil.ParseGeneric(version) - if err != nil { - return nil, false - } - - if ks, ok := StabledVersionSupport(fmt.Sprintf("v%s", v.String())); ok { - return ks, true - } - - return nil, false -} - -func Latest() *KsInstaller { - return VersionMap[VersionList[len(VersionList)-1].String()] -} diff --git a/cmd/kk/pkg/version/kubesphere/version_enum_test.go b/cmd/kk/pkg/version/kubesphere/version_enum_test.go deleted file mode 100644 index 5c7710b08..000000000 --- a/cmd/kk/pkg/version/kubesphere/version_enum_test.go +++ /dev/null @@ -1,242 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubesphere - -import ( - "reflect" - "testing" -) - -func TestDevRelease(t *testing.T) { - tests := []struct { - name string - version string - want *KsInstaller - ok bool - }{ - { - name: "test_v3.2.1-rc.1", - version: "v3.2.1-rc.1", - want: KsV321, - ok: true, - }, - { - name: "test_v3.2.1", - version: "v3.2.1", - want: nil, - ok: false, - }, - { - name: "test_v3.2.0", - version: "v3.2.0", - want: nil, - ok: false, - }, - { - name: "test_v3.2.0-alpha.1", - version: "v3.2.0-alpha.1", - want: KsV320, - ok: true, - }, - { - name: "test_v3.2.0-beta.1", - version: "v3.2.0-beta.1", - want: KsV320, - ok: true, - }, - { - name: "test_v3.1.0-alpha.1", - version: "v3.1.0-alpha.1", - want: KsV310, - ok: true, - }, - { - name: "test_latest", - version: "latest", - want: KsV331, - ok: true, - }, - { - name: "test_master", - version: "master", - want: KsV331, - ok: true, - }, - { - name: "test_release-3.3", - version: "release-3.3", - want: KsV331, - ok: true, - }, - { - name: "test_v1.2.0", - version: "v1.2.0", - want: nil, - ok: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, ok := DevRelease(tt.version) - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("DevRelease() got = %v, want %v", got, tt.want) - } - if ok != tt.ok { - t.Errorf("DevRelease() got1 = %v, want %v", ok, tt.ok) - } - }) - } -} - -func TestLatest(t *testing.T) { - tests := []struct { - name string - want *KsInstaller - }{ - { - name: "test_latest", - want: KsV331, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := Latest(); !reflect.DeepEqual(got, tt.want) { - t.Errorf("Latest() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestLatestRelease(t *testing.T) { - tests := []struct { - name string - version string - want *KsInstaller - ok bool - }{ - { - name: "test_latest", - version: "latest", - want: KsV331, - ok: true, - }, - { - name: "test_master", - version: "master", - want: KsV331, - ok: true, - }, - { - name: "test_release-3.2", - version: "release-3.2", - want: KsV331, - ok: true, - }, - { - name: "test_v3.3.1", - version: "v3.3.1", - want: KsV331, - ok: true, - }, - { - name: "test_v3.3.1-rc.1", - version: "v3.3.1-rc.1", - want: KsV331, - ok: true, - }, - { - name: "test_v3.2.0", - version: "v3.2.0", - want: nil, - ok: false, - }, - { - name: "test_v3.1.0", - version: "v3.1.0", - want: nil, - ok: false, - }, - { - name: "test_v3.2.0-alpha.1", - version: "v3.2.0-alpha.1", - want: nil, - ok: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, ok := LatestRelease(tt.version) - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("LatestRelease() got = %v, want %v", got, tt.want) - } - if ok != tt.ok { - t.Errorf("LatestRelease() got1 = %v, want %v", ok, tt.ok) - } - }) - } -} - -func TestStabledVersionSupport(t *testing.T) { - tests := []struct { - name string - version string - want *KsInstaller - ok bool - }{ - { - name: "test_v3.2.1-rc.1", - version: "v3.2.1-rc.1", - want: nil, - ok: false, - }, - { - name: "test_v3.2.0", - version: "v3.2.0", - want: KsV320, - ok: true, - }, - { - name: "test_3.2.0", - version: "3.2.0", - want: nil, - ok: false, - }, - { - name: "test_v3.2.0-alpha.1", - version: "v3.2.0-alpha.1", - want: nil, - ok: false, - }, - { - name: "test_v1.2.0", - version: "v1.2.0", - want: nil, - ok: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, ok := StabledVersionSupport(tt.version) - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("StabledVersionSupport() got = %v, want %v", got, tt.want) - } - if ok != tt.ok { - t.Errorf("StabledVersionSupport() got1 = %v, want %v", ok, tt.ok) - } - }) - } -} diff --git a/config/capkk/crds/infrastructure.cluster.x-k8s.io_kkclusters.yaml b/config/capkk/crds/infrastructure.cluster.x-k8s.io_kkclusters.yaml new file mode 100644 index 000000000..497a0fa2b --- /dev/null +++ b/config/capkk/crds/infrastructure.cluster.x-k8s.io_kkclusters.yaml @@ -0,0 +1,174 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.2-0.20250203100058-d1f73b5ea552 + labels: + cluster.x-k8s.io/v1beta1: v1beta1 + name: kkclusters.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: KKCluster + listKind: KKClusterList + plural: kkclusters + shortNames: + - kkc + singular: kkcluster + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Cluster to which this KKClusters belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + type: string + - description: Cluster infrastructure is ready for SSH instances + jsonPath: .status.ready + name: Ready + type: string + - description: the ControlPlaneEndpointType to connect workload cluster + jsonPath: .spec.controlPlaneEndpointType + name: ControlPlaneEndpointType + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: KKCluster resource maps a kubernetes cluster, manage and reconcile + cluster status. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: KKClusterSpec defines the desired state of KKCluster. + properties: + controlPlaneEndpointType: + description: |- + ControlPlaneEndpointType defines the type of control plane endpoint. such as dns, vip. + when use vip, it will deploy kube-vip in each control_plane node. the default value is vip. + type: string + hostCheckGroup: + description: |- + which Group defined in Inventory will be checked. there is some default group by system: + - all: contains all hosts + - ungrouped: contains hosts which do not belong to any groups. + if the value is empty, "ungrouped" will be used. + type: string + inventory: + description: InventoryHosts contains all hosts of the cluster. + items: + properties: + connector: + description: Connector to connect the host. + properties: + host: + description: Host address. default use host.name. + type: string + password: + description: Password is the password of the host. + type: string + privateKey: + description: PrivateKey is the private key of the host. + default is ~/.ssh/id_rsa. + type: string + type: + description: Type to connector the host. + type: string + user: + description: User is the user name of the host. default + is root. + type: string + type: object + name: + description: Name of the host. + type: string + vars: + description: Vars for the host. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: array + tolerate: + description: tolerate defines if tolerate host check if failed. + type: boolean + type: object + status: + description: KKClusterStatus defines the observed state of KKCluster. + properties: + conditions: + description: Conditions defines current service state of the KKCluster. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + A human readable message indicating details about the transition. + This field may be empty. + type: string + reason: + description: |- + The reason for the condition's last transition in CamelCase. + The specific API may choose whether or not this field is considered a guaranteed API. + This field may be empty. + type: string + severity: + description: |- + severity provides an explicit classification of Reason code, so the users or machines can immediately + understand the current situation and act accordingly. + The Severity field MUST be set only when Status=False. + type: string + status: + description: status of the condition, one of True, False, Unknown. + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions + can be useful (see .node.status.conditions), the ability to deconflict is important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + failureMessage: + type: string + failureReason: + description: FailureReason + type: string + ready: + description: if Ready to create cluster. usage after inventory is + ready. + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/capkk/crds/infrastructure.cluster.x-k8s.io_kkmachines.yaml b/config/capkk/crds/infrastructure.cluster.x-k8s.io_kkmachines.yaml new file mode 100644 index 000000000..a835508d1 --- /dev/null +++ b/config/capkk/crds/infrastructure.cluster.x-k8s.io_kkmachines.yaml @@ -0,0 +1,174 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.2-0.20250203100058-d1f73b5ea552 + labels: + cluster.x-k8s.io/v1beta1: v1beta1 + name: kkmachines.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: KKMachine + listKind: KKMachineList + plural: kkmachines + shortNames: + - kkm + singular: kkmachine + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Cluster to which this KKMachine belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + type: string + - description: the providerID for the machine + jsonPath: .spec.providerID + name: ProviderID + type: string + - description: Machine ready status + jsonPath: .status.ready + name: Ready + type: string + - description: Machine object which owns with this KKMachine + jsonPath: .metadata.ownerReferences[?(@.kind=="Machine")].name + name: Machine + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: KKMachine resource maps a machine instance, manage and reconcile + machine status. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: KKMachineSpec defines the desired state of KKMachine. + properties: + config: + description: Config for machine. contains cluster version, binary + version, etc. + type: object + x-kubernetes-preserve-unknown-fields: true + failureDomain: + description: |- + failureDomain is the failure domain the machine will be created in. + Must match a key in the FailureDomains map stored on the cluster object. + type: string + providerID: + description: |- + providerID is the identification ID of the machine provided by the provider. + This field must match the provider ID as seen on the node object corresponding to this machine. + This field is required by higher level consumers of cluster-api. Example use case is cluster autoscaler + with cluster-api as provider. Clean-up logic in the autoscaler compares machines to nodes to find out + machines at provider which could not get registered as Kubernetes nodes. With cluster-api as a + generic out-of-tree provider for autoscaler, this field is required by autoscaler to be + able to have a provider view of the list of machines. Another list of nodes is queried from the k8s apiserver + and then a comparison is done to find out unregistered machines and are marked for delete. + This field will be set by the actuators and consumed by higher level entities like autoscaler that will + be interfacing with cluster-api as generic provider. + type: string + roles: + description: |- + Roles defines the roles assigned to the Kubernetes cluster node, such as "worker" or "control-plane". + A KKMachine created by ControlPlane will automatically have the "control-plane" role. + A KKMachine created by MachineDeployment will automatically have the "worker" role. + Additional custom roles can also be specified in this field as needed. + items: + type: string + type: array + version: + description: |- + version defines the desired Kubernetes version. + This field is meant to be optionally used by bootstrap providers. + type: string + type: object + status: + description: KKMachineStatus defines the observed state of KKMachine. + properties: + certificatesExpiryDate: + description: |- + certificatesExpiryDate is the expiry date of the machine certificates. + This value is only set for control plane machines. + format: date-time + type: string + conditions: + description: Conditions defines current service state of the KKMachine. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + A human readable message indicating details about the transition. + This field may be empty. + type: string + reason: + description: |- + The reason for the condition's last transition in CamelCase. + The specific API may choose whether or not this field is considered a guaranteed API. + This field may be empty. + type: string + severity: + description: |- + severity provides an explicit classification of Reason code, so the users or machines can immediately + understand the current situation and act accordingly. + The Severity field MUST be set only when Status=False. + type: string + status: + description: status of the condition, one of True, False, Unknown. + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions + can be useful (see .node.status.conditions), the ability to deconflict is important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + failureMessage: + description: FailureMessage will be set in the event that there is + a terminal problem + type: string + failureReason: + description: FailureReason will be set in the event that there is + a terminal problem + type: string + ready: + description: Ready is true when the provider resource is ready. + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/capkk/crds/infrastructure.cluster.x-k8s.io_kkmachinetemplates.yaml b/config/capkk/crds/infrastructure.cluster.x-k8s.io_kkmachinetemplates.yaml new file mode 100644 index 000000000..b925bfba9 --- /dev/null +++ b/config/capkk/crds/infrastructure.cluster.x-k8s.io_kkmachinetemplates.yaml @@ -0,0 +1,132 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.2-0.20250203100058-d1f73b5ea552 + labels: + cluster.x-k8s.io/v1beta1: v1beta1 + name: kkmachinetemplates.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: KKMachineTemplate + listKind: KKMachineTemplateList + plural: kkmachinetemplates + shortNames: + - kkmt + singular: kkmachinetemplate + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Time duration since creation of KKMachineTemplate + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: KKMachineTemplate is the Schema for the kkmachinetemplates API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: KKMachineTemplateSpec defines the desired state of KKMachineTemplate. + properties: + template: + description: KKMachineTemplateResource describes the data needed to + create a KKMachine from a template. + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + properties: + annotations: + additionalProperties: + type: string + description: |- + annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + spec: + description: Spec is the specification of the desired behavior + of the machine. + properties: + config: + description: Config for machine. contains cluster version, + binary version, etc. + type: object + x-kubernetes-preserve-unknown-fields: true + failureDomain: + description: |- + failureDomain is the failure domain the machine will be created in. + Must match a key in the FailureDomains map stored on the cluster object. + type: string + providerID: + description: |- + providerID is the identification ID of the machine provided by the provider. + This field must match the provider ID as seen on the node object corresponding to this machine. + This field is required by higher level consumers of cluster-api. Example use case is cluster autoscaler + with cluster-api as provider. Clean-up logic in the autoscaler compares machines to nodes to find out + machines at provider which could not get registered as Kubernetes nodes. With cluster-api as a + generic out-of-tree provider for autoscaler, this field is required by autoscaler to be + able to have a provider view of the list of machines. Another list of nodes is queried from the k8s apiserver + and then a comparison is done to find out unregistered machines and are marked for delete. + This field will be set by the actuators and consumed by higher level entities like autoscaler that will + be interfacing with cluster-api as generic provider. + type: string + roles: + description: |- + Roles defines the roles assigned to the Kubernetes cluster node, such as "worker" or "control-plane". + A KKMachine created by ControlPlane will automatically have the "control-plane" role. + A KKMachine created by MachineDeployment will automatically have the "worker" role. + Additional custom roles can also be specified in this field as needed. + items: + type: string + type: array + version: + description: |- + version defines the desired Kubernetes version. + This field is meant to be optionally used by bootstrap providers. + type: string + type: object + required: + - spec + type: object + required: + - template + type: object + type: object + served: true + storage: true + subresources: {} diff --git a/config/capkk/crds/kubekey.kubesphere.io_inventories.yaml b/config/capkk/crds/kubekey.kubesphere.io_inventories.yaml new file mode 100644 index 000000000..9670c0d86 --- /dev/null +++ b/config/capkk/crds/kubekey.kubesphere.io_inventories.yaml @@ -0,0 +1,95 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.2-0.20250203100058-d1f73b5ea552 + name: inventories.kubekey.kubesphere.io +spec: + group: kubekey.kubesphere.io + names: + kind: Inventory + listKind: InventoryList + plural: inventories + singular: inventory + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Status of inventory + jsonPath: .status.phase + name: Phase + type: string + name: v1 + schema: + openAPIV3Schema: + description: Inventory store hosts vars for playbook. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: InventorySpec of Inventory + properties: + groups: + additionalProperties: + description: InventoryGroup of Inventory + properties: + groups: + items: + type: string + type: array + hosts: + items: + type: string + type: array + vars: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - hosts + type: object + description: Groups nodes. a group contains repeated nodes + type: object + hosts: + additionalProperties: + type: object + x-kubernetes-preserve-unknown-fields: true + description: Hosts is all nodes + type: object + vars: + description: 'Vars for all host. the priority for vars is: host vars + > group vars > inventory vars' + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - hosts + type: object + status: + description: InventoryStatus of Inventory + properties: + phase: + description: Phase is the inventory phase. + type: string + ready: + description: Ready is the inventory ready to be used. + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/capkk/crds/kubekey.kubesphere.io_playbooks.yaml b/config/capkk/crds/kubekey.kubesphere.io_playbooks.yaml new file mode 100644 index 000000000..5521c9bce --- /dev/null +++ b/config/capkk/crds/kubekey.kubesphere.io_playbooks.yaml @@ -0,0 +1,2034 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.2-0.20250203100058-d1f73b5ea552 + name: playbooks.kubekey.kubesphere.io +spec: + group: kubekey.kubesphere.io + names: + kind: Playbook + listKind: PlaybookList + plural: playbooks + singular: playbook + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.playbook + name: Playbook + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.taskResult.total + name: Total + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: |- + Playbook is the Schema for the playbooks API. + Playbook resource executor a playbook. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: PlaybookSpec defines the desired state of Playbook. + properties: + config: + description: Config is the global variable configuration for playbook + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + inventoryRef: + description: InventoryRef is the node configuration for playbook + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + playbook: + description: Playbook which to execute. + type: string + project: + description: Project is storage for executable packages + properties: + addr: + description: |- + Addr is the storage for executable packages (in Ansible file format). + When starting with http or https, it will be obtained from a Git repository. + When starting with file path, it will be obtained from the local path. + type: string + branch: + description: Branch is the git branch of the git Addr. + type: string + insecureSkipTLS: + description: InsecureSkipTLS skip tls or not when git addr is + https. + type: boolean + name: + description: Name is the project name base project + type: string + tag: + description: Tag is the git branch of the git Addr. + type: string + token: + description: Token of Authorization for http request + type: string + type: object + serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + type: string + skipTags: + description: SkipTags is the tags of playbook which skip execute + items: + type: string + type: array + tags: + description: Tags is the tags of playbook which to execute + items: + type: string + type: array + volumeMounts: + description: VolumeMounts in job pod. + items: + description: VolumeMount describes a mounting of a Volume within + a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + workVolume: + description: Volumes in job pod. + items: + description: Volume represents a named volume in a pod that may + be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure Data Disk mount on + the host and bind mount to the pod. + properties: + cachingMode: + description: 'cachingMode is the Host Caching mode: None, + Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data disk in the + blob storage + type: string + diskURI: + description: diskURI is the URI of data disk in the blob + storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: multiple + blob disks per storage account Dedicated: single blob + disk per storage account Managed: azure managed data + disk (only in managed availability set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File Service mount + on the host and bind mount to the pod. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret that contains + Azure Storage Account Name and Key + type: string + shareName: + description: shareName is the azure share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount on the host that + shares a pod's lifetime + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used as the mounted root, + rather than the full Ceph tree, default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap that should populate + this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap or its + keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) represents ephemeral + storage that is handled by certain external CSI drivers (Beta + feature). + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API about the pod + that should populate this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward API volume file + items: + description: DownwardAPIVolumeFile represents information + to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the pod: + only annotations, labels, name, namespace and uid + are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative path + name of the file to be created. Must not be absolute + or contain the ''..'' path. Must be utf-8 encoded. + The first item of the relative path must not start + with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query over volumes + to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference + to the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource that is + attached to a kubelet's host machine and then exposed to the + pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target worldwide + names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the driver to use for + this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field holds extra + command options if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume attached to + a kubelet's host machine. This depends on the Flocker control + service being running + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the dataset. This + is unique identifier of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for the specified + revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether support iSCSI + Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether support iSCSI + Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for iSCSI target + and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies Photon Controller + persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx volume attached + and mounted on kubelets host machine + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources secrets, + configmaps, and downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from the volume root + to write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information about the configMap + data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information about the downwardAPI + data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name, namespace and uid are supported.' + properties: + apiVersion: + description: Version of the schema the + FieldPath is written in terms of, + defaults to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. Must + not be absolute or contain the ''..'' + path. Must be utf-8 encoded. The first + item of the relative path must not start + with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults + to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information about the secret data + to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify whether the + Secret or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is information about + the serviceAccountToken data to project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: quobyte represents a Quobyte mount on the host + that shares a pod's lifetime + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references an already + created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address of the ScaleIO + API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of the ScaleIO + Protection Domain for the configured storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable SSL communication + with Gateway, default false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage Pool associated + with the protection domain. + type: string + system: + description: system is the name of the storage system as + configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify whether the Secret or + its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: storageOS represents a StorageOS volume attached + and mounted on Kubernetes nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere volume attached + and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage Policy Based + Management (SPBM) profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage Policy Based + Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies vSphere + volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + required: + - playbook + type: object + status: + description: PlaybookStatus defines the observed state of Playbook. + properties: + failureMessage: + description: FailureMessage will be set in the event that there is + a terminal problem + type: string + failureReason: + description: FailureReason will be set in the event that there is + a terminal problem + type: string + phase: + description: Phase of playbook. + type: string + result: + description: Result will record the results detail. + type: object + x-kubernetes-preserve-unknown-fields: true + statistics: + description: Statistics statistics of task counts + properties: + failed: + description: Number of failed tasks + type: integer + ignored: + description: Number of ignored tasks + type: integer + success: + description: Number of successful tasks + type: integer + total: + description: Total number of tasks + type: integer + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/capkk/deployment/capkk-controller-manager.yaml b/config/capkk/deployment/capkk-controller-manager.yaml new file mode 100644 index 000000000..b6f0f1816 --- /dev/null +++ b/config/capkk/deployment/capkk-controller-manager.yaml @@ -0,0 +1,75 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: capkk-controller-manager +spec: + replicas: 1 + selector: + matchLabels: + cluster.x-k8s.io/provider: infrastructure-kk + app: capkk-controller-manager + template: + metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-kk + app: capkk-controller-manager + spec: + containers: + - name: controller-manager + image: capkk-controller-manager-image + imagePullPolicy: IfNotPresent + command: + - controller-manager + - --leader-election + - --leader-election-id=controller-leader-election-capkk + - --v=4 + resources: + requests: + cpu: 20m + memory: 100Mi + limits: + cpu: 1 + memory: 1024Mi + livenessProbe: + httpGet: + path: /healthz + port: 9440 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 9440 + initialDelaySeconds: 5 + periodSeconds: 10 + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + securityContext: + allowPrivilegeEscalation: false + serviceAccountName: capkk + terminationGracePeriodSeconds: 10 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: capkk-webhook-service-cert + +--- +apiVersion: v1 +kind: Service +metadata: + name: capkk-webhook-service + namespace: capkk-system +spec: + ports: + - port: 443 + targetPort: 9443 + selector: + app: capkk-controller-manager \ No newline at end of file diff --git a/config/capkk/deployment/kk-controller-manager.yaml b/config/capkk/deployment/kk-controller-manager.yaml new file mode 100644 index 000000000..88023b83e --- /dev/null +++ b/config/capkk/deployment/kk-controller-manager.yaml @@ -0,0 +1,80 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kk-controller-manager +spec: + replicas: 1 + selector: + matchLabels: + cluster.x-k8s.io/provider: infrastructure-kk + app: kk-controller-manager + template: + metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-kk + app: kk-controller-manager + spec: + containers: + - name: controller-manager + image: kk-controller-manager-image + imagePullPolicy: IfNotPresent + command: + - controller-manager + - --leader-election + - --leader-election-id=controller-leader-election-kk + - --v=4 + env: + - name: EXECUTOR_IMAGE + value: docker.io/kubesphere/executor:latest + - name: EXECUTOR_CLUSTERROLE + value: capkk + resources: + requests: + cpu: 20m + memory: 100Mi + limits: + cpu: 1 + memory: 1024Mi + livenessProbe: + httpGet: + path: /healthz + port: 9440 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 9440 + initialDelaySeconds: 5 + periodSeconds: 10 + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + securityContext: + allowPrivilegeEscalation: false + serviceAccountName: capkk + terminationGracePeriodSeconds: 10 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: capkk-webhook-service-cert + +--- +apiVersion: v1 +kind: Service +metadata: + name: kk-webhook-service + namespace: capkk-system +spec: + ports: + - port: 443 + targetPort: 9443 + selector: + app: kk-controller-manager \ No newline at end of file diff --git a/config/capkk/kustomization.yaml b/config/capkk/kustomization.yaml new file mode 100644 index 000000000..fe52e8e75 --- /dev/null +++ b/config/capkk/kustomization.yaml @@ -0,0 +1,26 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: capkk-system + +resources: +- namespace.yaml +- crds/infrastructure.cluster.x-k8s.io_kkclusters.yaml +- crds/infrastructure.cluster.x-k8s.io_kkmachines.yaml +- crds/infrastructure.cluster.x-k8s.io_kkmachinetemplates.yaml +- crds/kubekey.kubesphere.io_inventories.yaml +- crds/kubekey.kubesphere.io_playbooks.yaml +- webhook/manifests.yaml +- webhook/issuer.yaml +- rbac/role.yaml +- rbac/serviceaccount.yaml +- deployment/kk-controller-manager.yaml +- deployment/capkk-controller-manager.yaml + +# Optional: Add common labels to all resources +commonLabels: + cluster.x-k8s.io/provider: infrastructure-kk + +images: +- name: capkk-controller-manager-image +- name: kk-controller-manager-image diff --git a/config/capkk/namespace.yaml b/config/capkk/namespace.yaml new file mode 100644 index 000000000..002429797 --- /dev/null +++ b/config/capkk/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: capkk-system diff --git a/config/capkk/rbac/role.yaml b/config/capkk/rbac/role.yaml new file mode 100644 index 000000000..1fddbebfe --- /dev/null +++ b/config/capkk/rbac/role.yaml @@ -0,0 +1,75 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: capkk +rules: +- apiGroups: + - "" + resources: + - events + - pods + - serviceaccounts + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - cluster.x-k8s.io + - controlplane.cluster.x-k8s.io + resources: + - '*' + verbs: + - get + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + - kubekey.kubesphere.io + resources: + - '*' + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterrolebindings + - clusterroles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch diff --git a/config/capkk/rbac/serviceaccount.yaml b/config/capkk/rbac/serviceaccount.yaml new file mode 100644 index 000000000..4982f0061 --- /dev/null +++ b/config/capkk/rbac/serviceaccount.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: capkk + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: capkk +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: capkk +subjects: + - kind: ServiceAccount + name: capkk diff --git a/config/capkk/release/cluster-template.yaml b/config/capkk/release/cluster-template.yaml new file mode 100644 index 000000000..38e17a058 --- /dev/null +++ b/config/capkk/release/cluster-template.yaml @@ -0,0 +1,107 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: '${CLUSTER_NAME}' +spec: + controlPlaneEndpoint: + host: "${CONTROL_PLANE_ENDPOINT_HOST}" + port: ${CONTROL_PLANE_ENDPOINT_PORT=6443} + clusterNetwork: + services: + cidrBlocks: ${SERVICE_CIDR=["10.233.0.0/18"]} + pods: + cidrBlocks: ${POD_CIDR=["10.233.64.0/18"]} + serviceDomain: ${SERVICE_DOMAIN="cluster.local"} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: KKCluster + name: '${CLUSTER_NAME}' + controlPlaneRef: + kind: KubeadmControlPlane + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: KKCluster +metadata: + name: '${CLUSTER_NAME}' +spec: + controlPlaneEndpointType: ${CONTROL_PLANE_ENDPOINT_TYPE="kube_vip"} + inventory: ${INVENTORY_HOSTS} +--- +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + version: "${KUBERNETES_VERSION}" + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + machineTemplate: + infrastructureRef: + kind: KKMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" + kubeadmConfigSpec: + initConfiguration: + nodeRegistration: + criSocket: ${CRI_SOCKET=unix:///run/containerd/containerd.sock} + joinConfiguration: + nodeRegistration: + criSocket: ${CRI_SOCKET=unix:///run/containerd/containerd.sock} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: KKMachineTemplate +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: + roles: + - control-plane + - master +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + "cluster.x-k8s.io/cluster-name": "${CLUSTER_NAME}" + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: KKMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: KKMachineTemplate +metadata: + name: '${CLUSTER_NAME}-md-0' +spec: + template: + spec: + roles: + - worker +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + criSocket: ${CRI_SOCKET=unix:///run/containerd/containerd.sock} diff --git a/config/capkk/release/metadata.yaml b/config/capkk/release/metadata.yaml new file mode 100644 index 000000000..cb1109946 --- /dev/null +++ b/config/capkk/release/metadata.yaml @@ -0,0 +1,13 @@ +# maps release series of major.minor to cluster-api contract version +# the contract version may change between minor or major versions, but *not* +# between patch versions. +# +# update this file only when a new major or minor version is released +apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 +releaseSeries: + - major: 4 + minor: 0 + contract: v1beta1 + - major: 3 + minor: 0 + contract: v1beta1 \ No newline at end of file diff --git a/config/capkk/webhook/issuer.yaml b/config/capkk/webhook/issuer.yaml new file mode 100644 index 000000000..854e1594c --- /dev/null +++ b/config/capkk/webhook/issuer.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: capkk-serving-cert + namespace: capkk-system +spec: + dnsNames: + - capkk-webhook-service.capkk-system + - capkk-webhook-service.capkk-system.svc + - kk-webhook-service.capkk-system + - kk-webhook-service.capkk-system.svc + issuerRef: + kind: Issuer + name: capkk-selfsigned-issuer + secretName: capkk-webhook-service-cert +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: capkk-selfsigned-issuer + namespace: capkk-system +spec: + selfSigned: {} \ No newline at end of file diff --git a/config/capkk/webhook/manifests.yaml b/config/capkk/webhook/manifests.yaml new file mode 100644 index 000000000..97edaf284 --- /dev/null +++ b/config/capkk/webhook/manifests.yaml @@ -0,0 +1,46 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: default-capkk +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kk-webhook-service + namespace: capkk-system + path: /mutate-kubekey-kubesphere-io-v1-playbook + failurePolicy: Fail + name: default.playbook.kubekey.kubesphere.io + rules: + - apiGroups: + - kubekey.kubesphere.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - playbooks + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: capkk-webhook-service + namespace: capkk-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-kkcluster + failurePolicy: Fail + name: default.kkcluster.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - kkclusters + sideEffects: None diff --git a/config/certmanager/certificate.yaml b/config/certmanager/certificate.yaml deleted file mode 100644 index 73a14e149..000000000 --- a/config/certmanager/certificate.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize - dnsNames: - - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc - - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: $(SERVICE_NAME)-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/config/certmanager/kustomization.yaml b/config/certmanager/kustomization.yaml deleted file mode 100644 index bebea5a59..000000000 --- a/config/certmanager/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -resources: -- certificate.yaml - -configurations: -- kustomizeconfig.yaml diff --git a/config/certmanager/kustomizeconfig.yaml b/config/certmanager/kustomizeconfig.yaml deleted file mode 100644 index d6109c9d8..000000000 --- a/config/certmanager/kustomizeconfig.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# This configuration is for teaching kustomize how to update name ref and var substitution -nameReference: -- kind: Issuer - group: cert-manager.io - fieldSpecs: - - kind: Certificate - group: cert-manager.io - path: spec/issuerRef/name - -varReference: -- kind: Certificate - group: cert-manager.io - path: spec/commonName -- kind: Certificate - group: cert-manager.io - path: spec/dnsNames -- kind: Certificate - group: cert-manager.io - path: spec/secretName diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_kkclusters.yaml deleted file mode 100644 index ba2bf021a..000000000 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkclusters.yaml +++ /dev/null @@ -1,409 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.1 - creationTimestamp: null - name: kkclusters.infrastructure.cluster.x-k8s.io -spec: - group: infrastructure.cluster.x-k8s.io - names: - categories: - - cluster-api - kind: KKCluster - listKind: KKClusterList - plural: kkclusters - shortNames: - - kkc - singular: kkcluster - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Cluster to which this KKClusters belongs - jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name - name: Cluster - type: string - - description: Cluster infrastructure is ready for SSH instances - jsonPath: .status.ready - name: Ready - type: string - - description: API Endpoint - jsonPath: .spec.controlPlaneEndpoint - name: Endpoint - priority: 1 - type: string - name: v1beta1 - schema: - openAPIV3Schema: - description: KKCluster is the Schema for the kkclusters API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: KKClusterSpec defines the desired state of KKCluster - properties: - component: - description: Component is optional configuration for modifying the - FTP server that gets the necessary components for the cluster. - properties: - host: - description: Host is the host to download the binaries. - type: string - overrides: - description: Overrides is a list of components download information - that need to be overridden. - items: - description: Override is a component download information that - need to be overridden. - properties: - arch: - description: Arch is the component arch. e.g. amd64, arm64, - etc. - type: string - checksum: - description: Checksum is the SHA256 checksum of the binary. - properties: - path: - description: Path defines the URL path, which is the - path of the checksum file. - type: string - value: - description: Value is the checksum string value. - type: string - type: object - id: - description: ID is the component id name. e.g. kubeadm, - kubelet, containerd, etc. - type: string - path: - description: Path defines the URL path, which is the string - of information that comes after the top level domain name. - type: string - url: - description: URL is the download url of the binaries. - type: string - version: - description: Version is the component version. e.g. v1.21.1, - v1.22.0, etc. - type: string - type: object - type: array - zone: - description: 'ZONE is the zone of the KKCluster where can get - the binaries. If you have problem to access https://storage.googleapis.com, - you can set "zone: cn".' - type: string - type: object - controlPlaneEndpoint: - description: ControlPlaneEndpoint represents the endpoint used to - communicate with the control plane. - properties: - host: - description: The hostname on which the API server is serving. - type: string - port: - description: The port on which the API server is serving. - format: int32 - type: integer - required: - - host - - port - type: object - controlPlaneLoadBalancer: - description: ControlPlaneLoadBalancer is optional configuration for - customizing control plane behavior. - properties: - host: - description: The hostname on which the API server is serving. - type: string - type: object - distribution: - description: Distribution represents the Kubernetes distribution type - of the cluster. - type: string - nodes: - description: Nodes represents the information about the nodes available - to the cluster - properties: - auth: - description: Auth is the SSH authentication information of all - instance. It is a global auth configuration. - properties: - password: - description: Password is the password for SSH authentication. - type: string - port: - description: Port is the port for SSH authentication. - type: integer - privateKey: - description: PrivateKey is the value of the private key for - SSH authentication. - type: string - privateKeyPath: - description: PrivateKeyFile is the path to the private key - for SSH authentication. - type: string - secret: - description: Secret is the secret of the PrivateKey or Password - for SSH authentication.It should in the same namespace as - capkk. When Password is empty, replace it with data.password. - When PrivateKey is empty, replace it with data.privateKey - type: string - timeout: - description: Timeout is the timeout for establish an SSH connection. - format: int64 - type: integer - user: - description: User is the username for SSH authentication. - type: string - type: object - instances: - description: Instances defines all instance contained in kkcluster. - items: - description: InstanceInfo defines the information about the - instance. - properties: - address: - description: Address is the IP address of the machine. - type: string - arch: - description: Arch is the architecture of the machine. e.g. - "amd64", "arm64". - type: string - auth: - description: Auth is the SSH authentication information - of this machine. It will override the global auth configuration. - properties: - password: - description: Password is the password for SSH authentication. - type: string - port: - description: Port is the port for SSH authentication. - type: integer - privateKey: - description: PrivateKey is the value of the private - key for SSH authentication. - type: string - privateKeyPath: - description: PrivateKeyFile is the path to the private - key for SSH authentication. - type: string - secret: - description: Secret is the secret of the PrivateKey - or Password for SSH authentication.It should in the - same namespace as capkk. When Password is empty, replace - it with data.password. When PrivateKey is empty, replace - it with data.privateKey - type: string - timeout: - description: Timeout is the timeout for establish an - SSH connection. - format: int64 - type: integer - user: - description: User is the username for SSH authentication. - type: string - type: object - internalAddress: - description: InternalAddress is the internal IP address - of the machine. - type: string - name: - description: Name is the host name of the machine. - minLength: 1 - type: string - roles: - description: Roles is the role of the machine. - items: - description: Role represents a role of a node. - type: string - type: array - type: object - type: array - required: - - instances - type: object - registry: - description: Registry represents the cluster image registry used to - pull the images. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this - representation of an object. Servers should convert recognized - schemas to the latest internal value, and may reject unrecognized - values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - auth: - description: Auth defines the auth of this PrivateRegistry. - properties: - caFile: - description: CAFile is an SSL Certificate Authority file used - to secure etcd communication. - type: string - certFile: - description: CertFile is an SSL certification file used to - secure etcd communication. - type: string - certsPath: - description: CertsPath defines the path of the certs files - of this PrivateRegistry. - type: string - insecureSkipVerify: - description: InsecureSkipVerify allow contacting this PrivateRegistry - over HTTPS with failed TLS verification. - type: boolean - keyFile: - description: KeyFile is an SSL key file used to secure etcd - communication. - type: string - password: - description: Password defines the password of this PrivateRegistry. - type: string - plainHTTP: - description: PlainHTTP allow contacting this PrivateRegistry - over HTTP. - type: boolean - username: - description: Username defines the username of this PrivateRegistry. - type: string - required: - - certsPath - - insecureSkipVerify - - password - - plainHTTP - - username - type: object - insecureRegistries: - description: InsecureRegistries defines the insecure registries - of ContainerManager. - items: - type: string - type: array - kind: - description: 'Kind is a string value representing the REST resource - this object represents. Servers may infer this from the endpoint - the client submits requests to. Cannot be updated. In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - namespaceOverride: - description: NamespaceOverride defines the namespace override - of this PrivateRegistry. - type: string - privateRegistry: - description: PrivateRegistry defines the private registry address - of ContainerManager. - type: string - registryMirrors: - description: RegistryMirrors defines the registry mirrors of this - PrivateRegistry. - items: - type: string - type: array - required: - - auth - - namespaceOverride - - privateRegistry - type: object - required: - - nodes - type: object - status: - description: KKClusterStatus defines the observed state of KKCluster - properties: - conditions: - description: Conditions defines current service state of the KKMachine. - items: - description: Condition defines an observation of a Cluster API resource - operational state. - properties: - lastTransitionTime: - description: Last time the condition transitioned from one status - to another. This should be when the underlying condition changed. - If that is not known, then using the time when the API field - changed is acceptable. - format: date-time - type: string - message: - description: A human readable message indicating details about - the transition. This field may be empty. - type: string - reason: - description: The reason for the condition's last transition - in CamelCase. The specific API may choose whether or not this - field is considered a guaranteed API. This field may not be - empty. - type: string - severity: - description: Severity provides an explicit classification of - Reason code, so the users or machines can immediately understand - the current situation and act accordingly. The Severity field - MUST be set only when Status=False. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of condition in CamelCase or in foo.example.com/CamelCase. - Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. - type: string - required: - - lastTransitionTime - - status - - type - type: object - type: array - failureMessage: - description: "FailureMessage will be set in the event that there is - a terminal problem reconciling the Machine and will contain a more - verbose string suitable for logging and human consumption. \n This - field should not be set for transitive errors that a controller - faces that are expected to be fixed automatically over time (like - service outages), but instead indicate that something is fundamentally - wrong with the Machine's spec or the configuration of the controller, - and that manual intervention is required. Examples of terminal errors - would be invalid combinations of settings in the spec, values that - are unsupported by the controller, or the responsible controller - itself being critically misconfigured. \n Any transient errors that - occur during the reconciliation of Machines can be added as events - to the Machine object and/or logged in the controller's output." - type: string - failureReason: - description: "FailureReason will be set in the event that there is - a terminal problem reconciling the Machine and will contain a succinct - value suitable for machine interpretation. \n This field should - not be set for transitive errors that a controller faces that are - expected to be fixed automatically over time (like service outages), - but instead indicate that something is fundamentally wrong with - the Machine's spec or the configuration of the controller, and that - manual intervention is required. Examples of terminal errors would - be invalid combinations of settings in the spec, values that are - unsupported by the controller, or the responsible controller itself - being critically misconfigured. \n Any transient errors that occur - during the reconciliation of Machines can be added as events to - the Machine object and/or logged in the controller's output." - type: string - ready: - default: false - type: boolean - required: - - ready - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_kkclustertemplates.yaml deleted file mode 100644 index 098b2aaf5..000000000 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkclustertemplates.yaml +++ /dev/null @@ -1,358 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.1 - creationTimestamp: null - name: kkclustertemplates.infrastructure.cluster.x-k8s.io -spec: - group: infrastructure.cluster.x-k8s.io - names: - categories: - - cluster-api - kind: KKClusterTemplate - listKind: KKClusterTemplateList - plural: kkclustertemplates - shortNames: - - kkct - singular: kkclustertemplate - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Time duration since creation of KKClusterTemplate - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: KKClusterTemplate is the Schema for the kkclustertemplates API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: KKClusterTemplateSpec defines the desired state of KKClusterTemplate - properties: - template: - description: KKClusterTemplateResource Standard object's metadata - properties: - metadata: - description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' - properties: - annotations: - additionalProperties: - type: string - description: 'Annotations is an unstructured key value map - stored with a resource that may be set by external tools - to store and retrieve arbitrary metadata. They are not queryable - and should be preserved when modifying objects. More info: - http://kubernetes.io/docs/user-guide/annotations' - type: object - labels: - additionalProperties: - type: string - description: 'Map of string keys and values that can be used - to organize and categorize (scope and select) objects. May - match selectors of replication controllers and services. - More info: http://kubernetes.io/docs/user-guide/labels' - type: object - type: object - spec: - description: KKClusterSpec defines the desired state of KKCluster - properties: - component: - description: Component is optional configuration for modifying - the FTP server that gets the necessary components for the - cluster. - properties: - host: - description: Host is the host to download the binaries. - type: string - overrides: - description: Overrides is a list of components download - information that need to be overridden. - items: - description: Override is a component download information - that need to be overridden. - properties: - arch: - description: Arch is the component arch. e.g. amd64, - arm64, etc. - type: string - checksum: - description: Checksum is the SHA256 checksum of - the binary. - properties: - path: - description: Path defines the URL path, which - is the path of the checksum file. - type: string - value: - description: Value is the checksum string value. - type: string - type: object - id: - description: ID is the component id name. e.g. kubeadm, - kubelet, containerd, etc. - type: string - path: - description: Path defines the URL path, which is - the string of information that comes after the - top level domain name. - type: string - url: - description: URL is the download url of the binaries. - type: string - version: - description: Version is the component version. e.g. - v1.21.1, v1.22.0, etc. - type: string - type: object - type: array - zone: - description: 'ZONE is the zone of the KKCluster where - can get the binaries. If you have problem to access - https://storage.googleapis.com, you can set "zone: cn".' - type: string - type: object - controlPlaneEndpoint: - description: ControlPlaneEndpoint represents the endpoint - used to communicate with the control plane. - properties: - host: - description: The hostname on which the API server is serving. - type: string - port: - description: The port on which the API server is serving. - format: int32 - type: integer - required: - - host - - port - type: object - controlPlaneLoadBalancer: - description: ControlPlaneLoadBalancer is optional configuration - for customizing control plane behavior. - properties: - host: - description: The hostname on which the API server is serving. - type: string - type: object - distribution: - description: Distribution represents the Kubernetes distribution - type of the cluster. - type: string - nodes: - description: Nodes represents the information about the nodes - available to the cluster - properties: - auth: - description: Auth is the SSH authentication information - of all instance. It is a global auth configuration. - properties: - password: - description: Password is the password for SSH authentication. - type: string - port: - description: Port is the port for SSH authentication. - type: integer - privateKey: - description: PrivateKey is the value of the private - key for SSH authentication. - type: string - privateKeyPath: - description: PrivateKeyFile is the path to the private - key for SSH authentication. - type: string - secret: - description: Secret is the secret of the PrivateKey - or Password for SSH authentication.It should in - the same namespace as capkk. When Password is empty, - replace it with data.password. When PrivateKey is - empty, replace it with data.privateKey - type: string - timeout: - description: Timeout is the timeout for establish - an SSH connection. - format: int64 - type: integer - user: - description: User is the username for SSH authentication. - type: string - type: object - instances: - description: Instances defines all instance contained - in kkcluster. - items: - description: InstanceInfo defines the information about - the instance. - properties: - address: - description: Address is the IP address of the machine. - type: string - arch: - description: Arch is the architecture of the machine. - e.g. "amd64", "arm64". - type: string - auth: - description: Auth is the SSH authentication information - of this machine. It will override the global auth - configuration. - properties: - password: - description: Password is the password for SSH - authentication. - type: string - port: - description: Port is the port for SSH authentication. - type: integer - privateKey: - description: PrivateKey is the value of the - private key for SSH authentication. - type: string - privateKeyPath: - description: PrivateKeyFile is the path to the - private key for SSH authentication. - type: string - secret: - description: Secret is the secret of the PrivateKey - or Password for SSH authentication.It should - in the same namespace as capkk. When Password - is empty, replace it with data.password. When - PrivateKey is empty, replace it with data.privateKey - type: string - timeout: - description: Timeout is the timeout for establish - an SSH connection. - format: int64 - type: integer - user: - description: User is the username for SSH authentication. - type: string - type: object - internalAddress: - description: InternalAddress is the internal IP - address of the machine. - type: string - name: - description: Name is the host name of the machine. - minLength: 1 - type: string - roles: - description: Roles is the role of the machine. - items: - description: Role represents a role of a node. - type: string - type: array - type: object - type: array - required: - - instances - type: object - registry: - description: Registry represents the cluster image registry - used to pull the images. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema - of this representation of an object. Servers should - convert recognized schemas to the latest internal value, - and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - auth: - description: Auth defines the auth of this PrivateRegistry. - properties: - caFile: - description: CAFile is an SSL Certificate Authority - file used to secure etcd communication. - type: string - certFile: - description: CertFile is an SSL certification file - used to secure etcd communication. - type: string - certsPath: - description: CertsPath defines the path of the certs - files of this PrivateRegistry. - type: string - insecureSkipVerify: - description: InsecureSkipVerify allow contacting this - PrivateRegistry over HTTPS with failed TLS verification. - type: boolean - keyFile: - description: KeyFile is an SSL key file used to secure - etcd communication. - type: string - password: - description: Password defines the password of this - PrivateRegistry. - type: string - plainHTTP: - description: PlainHTTP allow contacting this PrivateRegistry - over HTTP. - type: boolean - username: - description: Username defines the username of this - PrivateRegistry. - type: string - required: - - certsPath - - insecureSkipVerify - - password - - plainHTTP - - username - type: object - insecureRegistries: - description: InsecureRegistries defines the insecure registries - of ContainerManager. - items: - type: string - type: array - kind: - description: 'Kind is a string value representing the - REST resource this object represents. Servers may infer - this from the endpoint the client submits requests to. - Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - namespaceOverride: - description: NamespaceOverride defines the namespace override - of this PrivateRegistry. - type: string - privateRegistry: - description: PrivateRegistry defines the private registry - address of ContainerManager. - type: string - registryMirrors: - description: RegistryMirrors defines the registry mirrors - of this PrivateRegistry. - items: - type: string - type: array - required: - - auth - - namespaceOverride - - privateRegistry - type: object - required: - - nodes - type: object - required: - - spec - type: object - required: - - template - type: object - type: object - served: true - storage: true - subresources: {} diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkinstances.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_kkinstances.yaml deleted file mode 100644 index 1f6285b2e..000000000 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkinstances.yaml +++ /dev/null @@ -1,336 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.1 - creationTimestamp: null - name: kkinstances.infrastructure.cluster.x-k8s.io -spec: - group: infrastructure.cluster.x-k8s.io - names: - categories: - - cluster-api - kind: KKInstance - listKind: KKInstanceList - plural: kkinstances - shortNames: - - kki - singular: kkinstance - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: kubekey instance hostname - jsonPath: .spec.name - name: Hostname - type: string - - description: kubekey instance address - jsonPath: .spec.address - name: Address - type: string - - description: KKInstance state - jsonPath: .status.instanceState - name: State - type: string - - description: Time duration since creation of KKInstance - jsonPath: .metadata.creationTimestamp - name: Age - type: date - - description: Kubernetes version associated with this KKInstance - jsonPath: .status.nodeInfo.kubeletVersion - name: Version - type: string - name: v1beta1 - schema: - openAPIV3Schema: - description: KKInstance is the Schema for the kkinstances API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: KKInstanceSpec defines the desired state of KKInstance - properties: - address: - description: Address is the IP address of the machine. - type: string - arch: - description: Arch is the architecture of the machine. e.g. "amd64", - "arm64". - type: string - auth: - description: Auth is the SSH authentication information of this machine. - It will override the global auth configuration. - properties: - password: - description: Password is the password for SSH authentication. - type: string - port: - description: Port is the port for SSH authentication. - type: integer - privateKey: - description: PrivateKey is the value of the private key for SSH - authentication. - type: string - privateKeyPath: - description: PrivateKeyFile is the path to the private key for - SSH authentication. - type: string - secret: - description: Secret is the secret of the PrivateKey or Password - for SSH authentication.It should in the same namespace as capkk. - When Password is empty, replace it with data.password. When - PrivateKey is empty, replace it with data.privateKey - type: string - timeout: - description: Timeout is the timeout for establish an SSH connection. - format: int64 - type: integer - user: - description: User is the username for SSH authentication. - type: string - type: object - containerManager: - description: ContainerManager is the container manager config of this - machine. - properties: - criDockerdVersion: - description: CRIDockerdVersion defines the version of cri-dockerd, - available only when Type is docker. https://github.com/Mirantis/cri-dockerd - type: string - criSocket: - description: CRISocket is used to connect an existing CRIClient. - type: string - crictlVersion: - description: CRICTLVersion defines the version of CRICTL. - type: string - type: - description: Type defines the type of ContainerManager. "docker", - "containerd" - type: string - version: - description: Version defines the version of ContainerManager. - type: string - type: object - internalAddress: - description: InternalAddress is the internal IP address of the machine. - type: string - name: - description: Name is the host name of the machine. - minLength: 1 - type: string - repository: - description: Repository is the repository config of this machine. - properties: - iso: - description: 'ISO specifies the ISO file name. There are 3 options: - "": empty string means will not install the packages. "none": - no ISO file will be used. And capkk will use the default repository - to install the required packages. "auto": capkk will detect - the ISO file automatically. Only support Ubuntu/Debian/CentOS. - "xxx-20.04-debs-amd64.iso": use the specified name to get the - ISO file name.' - type: string - packages: - description: Packages is a list of packages to be installed. - items: - type: string - type: array - update: - description: Update will update the repository packages list and - cache if it is true. - type: boolean - type: object - roles: - description: Roles is the role of the machine. - items: - description: Role represents a role of a node. - type: string - type: array - type: object - status: - description: KKInstanceStatus defines the observed state of KKInstance - properties: - conditions: - description: Conditions defines current service state of the KKMachine. - items: - description: Condition defines an observation of a Cluster API resource - operational state. - properties: - lastTransitionTime: - description: Last time the condition transitioned from one status - to another. This should be when the underlying condition changed. - If that is not known, then using the time when the API field - changed is acceptable. - format: date-time - type: string - message: - description: A human readable message indicating details about - the transition. This field may be empty. - type: string - reason: - description: The reason for the condition's last transition - in CamelCase. The specific API may choose whether or not this - field is considered a guaranteed API. This field may not be - empty. - type: string - severity: - description: Severity provides an explicit classification of - Reason code, so the users or machines can immediately understand - the current situation and act accordingly. The Severity field - MUST be set only when Status=False. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of condition in CamelCase or in foo.example.com/CamelCase. - Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. - type: string - required: - - lastTransitionTime - - status - - type - type: object - type: array - failureMessage: - description: "FailureMessage will be set in the event that there is - a terminal problem reconciling the Machine and will contain a more - verbose string suitable for logging and human consumption. \n This - field should not be set for transitive errors that a controller - faces that are expected to be fixed automatically over time (like - service outages), but instead indicate that something is fundamentally - wrong with the Machine's spec or the configuration of the controller, - and that manual intervention is required. Examples of terminal errors - would be invalid combinations of settings in the spec, values that - are unsupported by the controller, or the responsible controller - itself being critically misconfigured. \n Any transient errors that - occur during the reconciliation of Machines can be added as events - to the Machine object and/or logged in the controller's output." - type: string - failureReason: - description: "FailureReason will be set in the event that there is - a terminal problem reconciling the Machine and will contain a succinct - value suitable for machine interpretation. \n This field should - not be set for transitive errors that a controller faces that are - expected to be fixed automatically over time (like service outages), - but instead indicate that something is fundamentally wrong with - the Machine's spec or the configuration of the controller, and that - manual intervention is required. Examples of terminal errors would - be invalid combinations of settings in the spec, values that are - unsupported by the controller, or the responsible controller itself - being critically misconfigured. \n Any transient errors that occur - during the reconciliation of Machines can be added as events to - the Machine object and/or logged in the controller's output." - type: string - instanceState: - description: The current state of the instance. - type: string - nodeInfo: - description: 'NodeInfo is a set of ids/uuids to uniquely identify - the node. More info: https://kubernetes.io/docs/concepts/nodes/node/#info' - properties: - architecture: - description: The Architecture reported by the node - type: string - bootID: - description: Boot ID reported by the node. - type: string - containerRuntimeVersion: - description: ContainerRuntime Version reported by the node through - runtime remote API (e.g. containerd://1.4.2). - type: string - kernelVersion: - description: Kernel Version reported by the node from 'uname -r' - (e.g. 3.16.0-0.bpo.4-amd64). - type: string - kubeProxyVersion: - description: KubeProxy Version reported by the node. - type: string - kubeletVersion: - description: Kubelet Version reported by the node. - type: string - machineID: - description: 'MachineID reported by the node. For unique machine - identification in the cluster this field is preferred. Learn - more from man(5) machine-id: http://man7.org/linux/man-pages/man5/machine-id.5.html' - type: string - operatingSystem: - description: The Operating System reported by the node - type: string - osImage: - description: OS Image reported by the node from /etc/os-release - (e.g. Debian GNU/Linux 7 (wheezy)). - type: string - systemUUID: - description: SystemUUID reported by the node. For unique machine - identification MachineID is preferred. This field is specific - to Red Hat hosts https://access.redhat.com/documentation/en-us/red_hat_subscription_management/1/html/rhsm/uuid - type: string - required: - - architecture - - bootID - - containerRuntimeVersion - - kernelVersion - - kubeProxyVersion - - kubeletVersion - - machineID - - operatingSystem - - osImage - - systemUUID - type: object - nodeRef: - description: NodeRef will point to the corresponding Node if it exists. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' - type: string - kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' - type: string - resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' - type: string - uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' - type: string - type: object - x-kubernetes-map-type: atomic - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkmachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_kkmachines.yaml deleted file mode 100644 index 9c022c4a6..000000000 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkmachines.yaml +++ /dev/null @@ -1,225 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.1 - creationTimestamp: null - name: kkmachines.infrastructure.cluster.x-k8s.io -spec: - group: infrastructure.cluster.x-k8s.io - names: - categories: - - cluster-api - kind: KKMachine - listKind: KKMachineList - plural: kkmachines - shortNames: - - kkm - singular: kkmachine - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Cluster to which this KKMachine belongs - jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name - name: Cluster - type: string - - description: KKInstance name - jsonPath: .spec.instanceID - name: Instance - type: string - - description: Machine ready status - jsonPath: .status.ready - name: Ready - type: string - - description: Machine object which owns with this KKMachine - jsonPath: .metadata.ownerReferences[?(@.kind=="Machine")].name - name: Machine - type: string - name: v1beta1 - schema: - openAPIV3Schema: - description: KKMachine is the Schema for the kkmachines API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: KKMachineSpec defines the desired state of KKMachine - properties: - containerManager: - description: ContainerManager is the container manager config of this - machine. - properties: - criDockerdVersion: - description: CRIDockerdVersion defines the version of cri-dockerd, - available only when Type is docker. https://github.com/Mirantis/cri-dockerd - type: string - criSocket: - description: CRISocket is used to connect an existing CRIClient. - type: string - crictlVersion: - description: CRICTLVersion defines the version of CRICTL. - type: string - type: - description: Type defines the type of ContainerManager. "docker", - "containerd" - type: string - version: - description: Version defines the version of ContainerManager. - type: string - type: object - instanceID: - description: InstanceID is the name of the KKInstance. - type: string - providerID: - description: ProviderID is the unique identifier as specified by the - kubekey provider. - type: string - repository: - description: Repository is the repository config of this machine. - properties: - iso: - description: 'ISO specifies the ISO file name. There are 3 options: - "": empty string means will not install the packages. "none": - no ISO file will be used. And capkk will use the default repository - to install the required packages. "auto": capkk will detect - the ISO file automatically. Only support Ubuntu/Debian/CentOS. - "xxx-20.04-debs-amd64.iso": use the specified name to get the - ISO file name.' - type: string - packages: - description: Packages is a list of packages to be installed. - items: - type: string - type: array - update: - description: Update will update the repository packages list and - cache if it is true. - type: boolean - type: object - roles: - description: Roles is the role of the machine. - items: - description: Role represents a role of a node. - type: string - type: array - type: object - status: - description: KKMachineStatus defines the observed state of KKMachine - properties: - addresses: - description: Addresses contains the KK instance associated addresses. - items: - description: MachineAddress contains information for the node's - address. - properties: - address: - description: The machine address. - type: string - type: - description: Machine address type, one of Hostname, ExternalIP - or InternalIP. - type: string - required: - - address - - type - type: object - type: array - conditions: - description: Conditions defines current service state of the KKMachine. - items: - description: Condition defines an observation of a Cluster API resource - operational state. - properties: - lastTransitionTime: - description: Last time the condition transitioned from one status - to another. This should be when the underlying condition changed. - If that is not known, then using the time when the API field - changed is acceptable. - format: date-time - type: string - message: - description: A human readable message indicating details about - the transition. This field may be empty. - type: string - reason: - description: The reason for the condition's last transition - in CamelCase. The specific API may choose whether or not this - field is considered a guaranteed API. This field may not be - empty. - type: string - severity: - description: Severity provides an explicit classification of - Reason code, so the users or machines can immediately understand - the current situation and act accordingly. The Severity field - MUST be set only when Status=False. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of condition in CamelCase or in foo.example.com/CamelCase. - Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. - type: string - required: - - lastTransitionTime - - status - - type - type: object - type: array - failureMessage: - description: "FailureMessage will be set in the event that there is - a terminal problem reconciling the Machine and will contain a more - verbose string suitable for logging and human consumption. \n This - field should not be set for transitive errors that a controller - faces that are expected to be fixed automatically over time (like - service outages), but instead indicate that something is fundamentally - wrong with the Machine's spec or the configuration of the controller, - and that manual intervention is required. Examples of terminal errors - would be invalid combinations of settings in the spec, values that - are unsupported by the controller, or the responsible controller - itself being critically misconfigured. \n Any transient errors that - occur during the reconciliation of Machines can be added as events - to the Machine object and/or logged in the controller's output." - type: string - failureReason: - description: "FailureReason will be set in the event that there is - a terminal problem reconciling the Machine and will contain a succinct - value suitable for machine interpretation. \n This field should - not be set for transitive errors that a controller faces that are - expected to be fixed automatically over time (like service outages), - but instead indicate that something is fundamentally wrong with - the Machine's spec or the configuration of the controller, and that - manual intervention is required. Examples of terminal errors would - be invalid combinations of settings in the spec, values that are - unsupported by the controller, or the responsible controller itself - being critically misconfigured. \n Any transient errors that occur - during the reconciliation of Machines can be added as events to - the Machine object and/or logged in the controller's output." - type: string - instanceState: - description: InstanceState is the state of the KK instance for this - machine. - type: string - ready: - description: Ready is true when the provider resource is ready. - type: boolean - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkmachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_kkmachinetemplates.yaml deleted file mode 100644 index 398a90529..000000000 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkmachinetemplates.yaml +++ /dev/null @@ -1,159 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.1 - creationTimestamp: null - name: kkmachinetemplates.infrastructure.cluster.x-k8s.io -spec: - group: infrastructure.cluster.x-k8s.io - names: - categories: - - cluster-api - kind: KKMachineTemplate - listKind: KKMachineTemplateList - plural: kkmachinetemplates - shortNames: - - kkmt - singular: kkmachinetemplate - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Time duration since creation of KKMachineTemplate - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: KKMachineTemplate is the Schema for the kkmachinetemplates API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: KKMachineTemplateSpec defines the desired state of KKMachineTemplate - properties: - template: - description: KKMachineTemplateResource describes the data needed to - create am KKMachine from a template. - properties: - metadata: - description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' - properties: - annotations: - additionalProperties: - type: string - description: 'Annotations is an unstructured key value map - stored with a resource that may be set by external tools - to store and retrieve arbitrary metadata. They are not queryable - and should be preserved when modifying objects. More info: - http://kubernetes.io/docs/user-guide/annotations' - type: object - labels: - additionalProperties: - type: string - description: 'Map of string keys and values that can be used - to organize and categorize (scope and select) objects. May - match selectors of replication controllers and services. - More info: http://kubernetes.io/docs/user-guide/labels' - type: object - type: object - spec: - description: Spec is the specification of the desired behavior - of the machine. - properties: - containerManager: - description: ContainerManager is the container manager config - of this machine. - properties: - criDockerdVersion: - description: CRIDockerdVersion defines the version of - cri-dockerd, available only when Type is docker. https://github.com/Mirantis/cri-dockerd - type: string - criSocket: - description: CRISocket is used to connect an existing - CRIClient. - type: string - crictlVersion: - description: CRICTLVersion defines the version of CRICTL. - type: string - type: - description: Type defines the type of ContainerManager. - "docker", "containerd" - type: string - version: - description: Version defines the version of ContainerManager. - type: string - type: object - instanceID: - description: InstanceID is the name of the KKInstance. - type: string - providerID: - description: ProviderID is the unique identifier as specified - by the kubekey provider. - type: string - repository: - description: Repository is the repository config of this machine. - properties: - iso: - description: 'ISO specifies the ISO file name. There are - 3 options: "": empty string means will not install the - packages. "none": no ISO file will be used. And capkk - will use the default repository to install the required - packages. "auto": capkk will detect the ISO file automatically. - Only support Ubuntu/Debian/CentOS. "xxx-20.04-debs-amd64.iso": - use the specified name to get the ISO file name.' - type: string - packages: - description: Packages is a list of packages to be installed. - items: - type: string - type: array - update: - description: Update will update the repository packages - list and cache if it is true. - type: boolean - type: object - roles: - description: Roles is the role of the machine. - items: - description: Role represents a role of a node. - type: string - type: array - type: object - required: - - spec - type: object - required: - - template - type: object - status: - description: KKMachineTemplateStatus defines a status for an KKMachineTemplate. - properties: - capacity: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Capacity defines the resource capacity for this machine. - This value is used for autoscaling from zero operations as defined - in: https://github.com/kubernetes-sigs/cluster-api/blob/main/docs/proposals/20210310-opt-in-autoscaling-from-zero.md' - type: object - type: object - type: object - served: true - storage: true - subresources: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml deleted file mode 100644 index f56714daa..000000000 --- a/config/crd/kustomization.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# This kustomization.yaml is not intended to be run by itself, -# since it depends on service name and namespace that are out of this kustomize package. -# It should be run by config/default -resources: -- bases/infrastructure.cluster.x-k8s.io_kkclusters.yaml -- bases/infrastructure.cluster.x-k8s.io_kkclustertemplates.yaml -- bases/infrastructure.cluster.x-k8s.io_kkmachines.yaml -- bases/infrastructure.cluster.x-k8s.io_kkmachinetemplates.yaml -- bases/infrastructure.cluster.x-k8s.io_kkinstances.yaml -#+kubebuilder:scaffold:crdkustomizeresource - -commonLabels: - cluster.x-k8s.io/v1beta1: v1beta1 - -patchesStrategicMerge: -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. -# patches here are for enabling the conversion webhook for each CRD -- patches/webhook_in_kkclusters.yaml -- patches/webhook_in_kkclustertemplates.yaml -- patches/webhook_in_kkmachines.yaml -- patches/webhook_in_kkmachinetemplates.yaml -- patches/webhook_in_kkinstances.yaml -#+kubebuilder:scaffold:crdkustomizewebhookpatch - -# [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. -# patches here are for enabling the CA injection for each CRD -- patches/cainjection_in_kkclusters.yaml -- patches/cainjection_in_kkclustertemplates.yaml -- patches/cainjection_in_kkmachines.yaml -- patches/cainjection_in_kkmachinetemplates.yaml -- patches/cainjection_in_kkinstances.yaml -#+kubebuilder:scaffold:crdkustomizecainjectionpatch - -# the following config is for teaching kustomize how to do kustomization for CRDs. -configurations: -- kustomizeconfig.yaml diff --git a/config/crd/kustomizeconfig.yaml b/config/crd/kustomizeconfig.yaml deleted file mode 100644 index ec5c150a9..000000000 --- a/config/crd/kustomizeconfig.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# This file is for teaching kustomize how to substitute name and namespace reference in CRD -nameReference: -- kind: Service - version: v1 - fieldSpecs: - - kind: CustomResourceDefinition - version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/name - -namespace: -- kind: CustomResourceDefinition - version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/namespace - create: false - -varReference: -- path: metadata/annotations diff --git a/config/crd/patches/cainjection_in_kkclusters.yaml b/config/crd/patches/cainjection_in_kkclusters.yaml deleted file mode 100644 index aabf0075d..000000000 --- a/config/crd/patches/cainjection_in_kkclusters.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: kkclusters.infrastructure.cluster.x-k8s.io diff --git a/config/crd/patches/cainjection_in_kkclustertemplates.yaml b/config/crd/patches/cainjection_in_kkclustertemplates.yaml deleted file mode 100644 index 4f99f8077..000000000 --- a/config/crd/patches/cainjection_in_kkclustertemplates.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: kkclustertemplates.infrastructure.cluster.x-k8s.io diff --git a/config/crd/patches/cainjection_in_kkinstances.yaml b/config/crd/patches/cainjection_in_kkinstances.yaml deleted file mode 100644 index e2b9e505b..000000000 --- a/config/crd/patches/cainjection_in_kkinstances.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: kkinstances.infrastructure.cluster.x-k8s.io diff --git a/config/crd/patches/cainjection_in_kkmachines.yaml b/config/crd/patches/cainjection_in_kkmachines.yaml deleted file mode 100644 index 30c406ee2..000000000 --- a/config/crd/patches/cainjection_in_kkmachines.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: kkmachines.infrastructure.cluster.x-k8s.io diff --git a/config/crd/patches/cainjection_in_kkmachinetemplates.yaml b/config/crd/patches/cainjection_in_kkmachinetemplates.yaml deleted file mode 100644 index 74c7289b0..000000000 --- a/config/crd/patches/cainjection_in_kkmachinetemplates.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: kkmachinetemplates.infrastructure.cluster.x-k8s.io diff --git a/config/crd/patches/webhook_in_kkclusters.yaml b/config/crd/patches/webhook_in_kkclusters.yaml deleted file mode 100644 index 3ca82219f..000000000 --- a/config/crd/patches/webhook_in_kkclusters.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: kkclusters.infrastructure.cluster.x-k8s.io -spec: - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: ["v1", "v1beta1"] - clientConfig: - # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, - # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) - caBundle: Cg== - service: - namespace: system - name: webhook-service - path: /convert diff --git a/config/crd/patches/webhook_in_kkclustertemplates.yaml b/config/crd/patches/webhook_in_kkclustertemplates.yaml deleted file mode 100644 index 466cb6225..000000000 --- a/config/crd/patches/webhook_in_kkclustertemplates.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: kkclustertemplates.infrastructure.cluster.x-k8s.io -spec: - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: ["v1", "v1beta1"] - clientConfig: - # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, - # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) - caBundle: Cg== - service: - namespace: system - name: webhook-service - path: /convert diff --git a/config/crd/patches/webhook_in_kkinstances.yaml b/config/crd/patches/webhook_in_kkinstances.yaml deleted file mode 100644 index e5be13f1c..000000000 --- a/config/crd/patches/webhook_in_kkinstances.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: kkinstances.infrastructure.cluster.x-k8s.io -spec: - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: ["v1", "v1beta1"] - clientConfig: - # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, - # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) - caBundle: Cg== - service: - namespace: system - name: webhook-service - path: /convert diff --git a/config/crd/patches/webhook_in_kkmachines.yaml b/config/crd/patches/webhook_in_kkmachines.yaml deleted file mode 100644 index 220581387..000000000 --- a/config/crd/patches/webhook_in_kkmachines.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: kkmachines.infrastructure.cluster.x-k8s.io -spec: - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: ["v1", "v1beta1"] - clientConfig: - # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, - # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) - caBundle: Cg== - service: - namespace: system - name: webhook-service - path: /convert diff --git a/config/crd/patches/webhook_in_kkmachinetemplates.yaml b/config/crd/patches/webhook_in_kkmachinetemplates.yaml deleted file mode 100644 index 008ed4a10..000000000 --- a/config/crd/patches/webhook_in_kkmachinetemplates.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: kkmachinetemplates.infrastructure.cluster.x-k8s.io -spec: - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: ["v1", "v1beta1"] - clientConfig: - # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, - # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) - caBundle: Cg== - service: - namespace: system - name: webhook-service - path: /convert diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml deleted file mode 100644 index 5e3f4334d..000000000 --- a/config/default/kustomization.yaml +++ /dev/null @@ -1,48 +0,0 @@ -namePrefix: capkk- -namespace: capkk-system - -commonLabels: - cluster.x-k8s.io/provider: "infrastructure-kk" - -bases: -- ../rbac -- ../manager -- ../crd -- ../certmanager -- ../webhook - -patchesStrategicMerge: -- manager_webhook_patch.yaml -- webhookcainjection_patch.yaml -- manager_pull_policy.yaml -- manager_image_patch.yaml - -configurations: - - kustomizeconfig.yaml -vars: - - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR - objref: - kind: Certificate - group: cert-manager.io - version: v1 - name: serving-cert # this name should match the one in certificate.yaml - fieldref: - fieldpath: metadata.namespace - - name: CERTIFICATE_NAME - objref: - kind: Certificate - group: cert-manager.io - version: v1 - name: serving-cert # this name should match the one in certificate.yaml - - name: SERVICE_NAMESPACE # namespace of the service - objref: - kind: Service - version: v1 - name: webhook-service - fieldref: - fieldpath: metadata.namespace - - name: SERVICE_NAME - objref: - kind: Service - version: v1 - name: webhook-service diff --git a/config/default/kustomizeconfig.yaml b/config/default/kustomizeconfig.yaml deleted file mode 100644 index 524d39cc2..000000000 --- a/config/default/kustomizeconfig.yaml +++ /dev/null @@ -1,4 +0,0 @@ -# This configuration is for teaching kustomize how to update name ref and var substitution -varReference: - - kind: Deployment - path: spec/template/spec/volumes/secret/secretName diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml deleted file mode 100644 index 1cba52d77..000000000 --- a/config/default/manager_image_patch.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - image: docker.io/kubespheredev/capkk-controller:main - name: manager diff --git a/config/default/manager_pull_policy.yaml b/config/default/manager_pull_policy.yaml deleted file mode 100644 index b010a1ee4..000000000 --- a/config/default/manager_pull_policy.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: manager - imagePullPolicy: IfNotPresent \ No newline at end of file diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml deleted file mode 100644 index b387eb0ea..000000000 --- a/config/default/manager_webhook_patch.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: manager - ports: - - containerPort: 9443 - name: webhook-server - protocol: TCP - volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true - volumes: - - name: cert - secret: - defaultMode: 420 - secretName: $(SERVICE_NAME)-cert diff --git a/config/default/webhookcainjection_patch.yaml b/config/default/webhookcainjection_patch.yaml deleted file mode 100644 index 02ab515d4..000000000 --- a/config/default/webhookcainjection_patch.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# This patch add annotation to admission webhook config and -# the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: mutating-webhook-configuration - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validating-webhook-configuration - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) diff --git a/config/file.go b/config/file.go new file mode 100644 index 000000000..4b000a0c1 --- /dev/null +++ b/config/file.go @@ -0,0 +1,9 @@ +package config + +import "embed" + +// Swagger embeds the swagger-ui directory containing the OpenAPI/Swagger documentation UI +// This allows serving the Swagger UI directly from the binary without needing external files +// +//go:embed swagger-ui +var Swagger embed.FS diff --git a/config/kubekey/Chart.yaml b/config/kubekey/Chart.yaml new file mode 100644 index 000000000..34a9ea8ed --- /dev/null +++ b/config/kubekey/Chart.yaml @@ -0,0 +1,15 @@ +apiVersion: v2 +name: kubekey +description: A Helm chart for kubekey + +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 1.0.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +appVersion: "dev" diff --git a/config/kubekey/crds/kubekey.kubesphere.io_inventories.yaml b/config/kubekey/crds/kubekey.kubesphere.io_inventories.yaml new file mode 100644 index 000000000..9670c0d86 --- /dev/null +++ b/config/kubekey/crds/kubekey.kubesphere.io_inventories.yaml @@ -0,0 +1,95 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.2-0.20250203100058-d1f73b5ea552 + name: inventories.kubekey.kubesphere.io +spec: + group: kubekey.kubesphere.io + names: + kind: Inventory + listKind: InventoryList + plural: inventories + singular: inventory + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Status of inventory + jsonPath: .status.phase + name: Phase + type: string + name: v1 + schema: + openAPIV3Schema: + description: Inventory store hosts vars for playbook. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: InventorySpec of Inventory + properties: + groups: + additionalProperties: + description: InventoryGroup of Inventory + properties: + groups: + items: + type: string + type: array + hosts: + items: + type: string + type: array + vars: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - hosts + type: object + description: Groups nodes. a group contains repeated nodes + type: object + hosts: + additionalProperties: + type: object + x-kubernetes-preserve-unknown-fields: true + description: Hosts is all nodes + type: object + vars: + description: 'Vars for all host. the priority for vars is: host vars + > group vars > inventory vars' + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - hosts + type: object + status: + description: InventoryStatus of Inventory + properties: + phase: + description: Phase is the inventory phase. + type: string + ready: + description: Ready is the inventory ready to be used. + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/kubekey/crds/kubekey.kubesphere.io_playbooks.yaml b/config/kubekey/crds/kubekey.kubesphere.io_playbooks.yaml new file mode 100644 index 000000000..5521c9bce --- /dev/null +++ b/config/kubekey/crds/kubekey.kubesphere.io_playbooks.yaml @@ -0,0 +1,2034 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.2-0.20250203100058-d1f73b5ea552 + name: playbooks.kubekey.kubesphere.io +spec: + group: kubekey.kubesphere.io + names: + kind: Playbook + listKind: PlaybookList + plural: playbooks + singular: playbook + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.playbook + name: Playbook + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.taskResult.total + name: Total + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: |- + Playbook is the Schema for the playbooks API. + Playbook resource executor a playbook. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: PlaybookSpec defines the desired state of Playbook. + properties: + config: + description: Config is the global variable configuration for playbook + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + inventoryRef: + description: InventoryRef is the node configuration for playbook + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + playbook: + description: Playbook which to execute. + type: string + project: + description: Project is storage for executable packages + properties: + addr: + description: |- + Addr is the storage for executable packages (in Ansible file format). + When starting with http or https, it will be obtained from a Git repository. + When starting with file path, it will be obtained from the local path. + type: string + branch: + description: Branch is the git branch of the git Addr. + type: string + insecureSkipTLS: + description: InsecureSkipTLS skip tls or not when git addr is + https. + type: boolean + name: + description: Name is the project name base project + type: string + tag: + description: Tag is the git branch of the git Addr. + type: string + token: + description: Token of Authorization for http request + type: string + type: object + serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + type: string + skipTags: + description: SkipTags is the tags of playbook which skip execute + items: + type: string + type: array + tags: + description: Tags is the tags of playbook which to execute + items: + type: string + type: array + volumeMounts: + description: VolumeMounts in job pod. + items: + description: VolumeMount describes a mounting of a Volume within + a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + workVolume: + description: Volumes in job pod. + items: + description: Volume represents a named volume in a pod that may + be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure Data Disk mount on + the host and bind mount to the pod. + properties: + cachingMode: + description: 'cachingMode is the Host Caching mode: None, + Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data disk in the + blob storage + type: string + diskURI: + description: diskURI is the URI of data disk in the blob + storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: multiple + blob disks per storage account Dedicated: single blob + disk per storage account Managed: azure managed data + disk (only in managed availability set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File Service mount + on the host and bind mount to the pod. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret that contains + Azure Storage Account Name and Key + type: string + shareName: + description: shareName is the azure share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount on the host that + shares a pod's lifetime + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used as the mounted root, + rather than the full Ceph tree, default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap that should populate + this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap or its + keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) represents ephemeral + storage that is handled by certain external CSI drivers (Beta + feature). + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API about the pod + that should populate this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward API volume file + items: + description: DownwardAPIVolumeFile represents information + to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the pod: + only annotations, labels, name, namespace and uid + are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative path + name of the file to be created. Must not be absolute + or contain the ''..'' path. Must be utf-8 encoded. + The first item of the relative path must not start + with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query over volumes + to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference + to the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource that is + attached to a kubelet's host machine and then exposed to the + pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target worldwide + names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the driver to use for + this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field holds extra + command options if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume attached to + a kubelet's host machine. This depends on the Flocker control + service being running + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the dataset. This + is unique identifier of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for the specified + revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether support iSCSI + Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether support iSCSI + Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for iSCSI target + and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies Photon Controller + persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx volume attached + and mounted on kubelets host machine + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources secrets, + configmaps, and downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from the volume root + to write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information about the configMap + data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information about the downwardAPI + data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name, namespace and uid are supported.' + properties: + apiVersion: + description: Version of the schema the + FieldPath is written in terms of, + defaults to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. Must + not be absolute or contain the ''..'' + path. Must be utf-8 encoded. The first + item of the relative path must not start + with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults + to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information about the secret data + to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify whether the + Secret or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is information about + the serviceAccountToken data to project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: quobyte represents a Quobyte mount on the host + that shares a pod's lifetime + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references an already + created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address of the ScaleIO + API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of the ScaleIO + Protection Domain for the configured storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable SSL communication + with Gateway, default false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage Pool associated + with the protection domain. + type: string + system: + description: system is the name of the storage system as + configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify whether the Secret or + its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: storageOS represents a StorageOS volume attached + and mounted on Kubernetes nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere volume attached + and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage Policy Based + Management (SPBM) profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage Policy Based + Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies vSphere + volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + required: + - playbook + type: object + status: + description: PlaybookStatus defines the observed state of Playbook. + properties: + failureMessage: + description: FailureMessage will be set in the event that there is + a terminal problem + type: string + failureReason: + description: FailureReason will be set in the event that there is + a terminal problem + type: string + phase: + description: Phase of playbook. + type: string + result: + description: Result will record the results detail. + type: object + x-kubernetes-preserve-unknown-fields: true + statistics: + description: Statistics statistics of task counts + properties: + failed: + description: Number of failed tasks + type: integer + ignored: + description: Number of ignored tasks + type: integer + success: + description: Number of successful tasks + type: integer + total: + description: Total number of tasks + type: integer + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/kubekey/templates/_helpers.tpl b/config/kubekey/templates/_helpers.tpl new file mode 100644 index 000000000..74356011a --- /dev/null +++ b/config/kubekey/templates/_helpers.tpl @@ -0,0 +1,45 @@ +{{/* +Common labels +*/}} +{{- define "common.labels" -}} +helm.sh/chart: {{ include "common.chart" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "common.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "operator.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.operator.image "global" .Values.global "chart" .Chart )}} +{{- end -}} +{{- define "executor.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.executor.image "global" .Values.global "chart" .Chart ) }} +{{- end -}} + +{{- define "common.images.image" -}} +{{- $registryName := .global.imageRegistry -}} +{{- $repositoryName := .imageRoot.repository -}} +{{- $separator := ":" -}} +{{- $termination := .chart.AppVersion | toString -}} +{{- if .global.tag }} +{{- $termination = .global.tag | toString -}} +{{- end -}} +{{- if .imageRoot.registry }} + {{- $registryName = .imageRoot.registry -}} +{{- end -}} +{{- if .imageRoot.tag }} + {{- $termination = .imageRoot.tag | toString -}} +{{- end -}} +{{- if .imageRoot.digest }} + {{- $separator = "@" -}} + {{- $termination = .imageRoot.digest | toString -}} +{{- end -}} +{{- printf "%s/%s%s%s" $registryName $repositoryName $separator $termination -}} +{{- end -}} diff --git a/config/kubekey/templates/_tplvalues.tpl b/config/kubekey/templates/_tplvalues.tpl new file mode 100644 index 000000000..2db166851 --- /dev/null +++ b/config/kubekey/templates/_tplvalues.tpl @@ -0,0 +1,13 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Renders a value that contains template. +Usage: +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $) }} +*/}} +{{- define "common.tplvalues.render" -}} + {{- if typeIs "string" .value }} + {{- tpl .value .context }} + {{- else }} + {{- tpl (.value | toYaml) .context }} + {{- end }} +{{- end -}} diff --git a/config/kubekey/templates/deployment.yaml b/config/kubekey/templates/deployment.yaml new file mode 100644 index 000000000..1cda1de79 --- /dev/null +++ b/config/kubekey/templates/deployment.yaml @@ -0,0 +1,96 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kk-controller-manager + namespace: {{ .Release.Namespace }} + labels: {{ include "common.labels" . | nindent 4 }} + app: kk-controller-manager +spec: + strategy: + rollingUpdate: + maxSurge: 0 + type: RollingUpdate + progressDeadlineSeconds: 600 + replicas: {{ .Values.controller-manager.replicaCount }} + revisionHistoryLimit: 10 + selector: + matchLabels: + app: kk-controller-manager + template: + metadata: + labels: {{ include "common.labels" . | nindent 8 }} + app: kk-controller-manager + spec: + serviceAccountName: kubekey + {{- if .Values.controller-manager.pullSecrets }} + imagePullSecrets: {{ .Values.controller-manager.pullSecrets }} + {{- end }} + {{- if .Values.controller-manager.nodeSelector }} + nodeSelector: {{ .Values.controller-manager.nodeSelector }} + {{- end }} + {{- if .Values.controller-manager.affinity }} + affinity: {{ .Values.controller-manager.affinity }} + {{- end }} + {{- if .Values.controller-manager.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.controller-manager.tolerations "context" .) | nindent 8 }} + {{- end }} + dnsPolicy: {{ .Values.controller-manager.dnsPolicy }} + restartPolicy: {{ .Values.controller-manager.restartPolicy }} + schedulerName: {{ .Values.controller-manager.schedulerName }} + terminationGracePeriodSeconds: {{ .Values.controller-manager.terminationGracePeriodSeconds }} + containers: + - name: controller-manager + image: {{ template "operator.image" . }} + imagePullPolicy: {{ .Values.controller-manager.image.pullPolicy }} + {{- if .Values.controller-manager.command | default list | len | lt 0 }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.controller-manager.command "context" $) | nindent 12 }} + {{- else }} + command: + - controller-manager + - --logtostderr=true + - --leader-election=true + {{-if not .Values.controller-manager.webhook.enabled }} + - --controllers=* + - --controllers=-playbook-webhook + {{- end }} + {{- end }} + env: + {{- if .Values.controller-manager.extraEnvVars }} + {{- include "common.tplvalues.render" (dict "value" .Values.controller-manager.extraEnvVars "context" $) | nindent 12 }} + {{- end }} + - name: EXECUTOR_IMAGE + value: {{ template "executor.image" . }} + - name: EXECUTOR_CLUSTERROLE + value: kubekey + - name: EXECUTOR_IMAGE_PULLPOLICY + value: {{ .Values.executor.image.pullPolicy }} + {{- if .Values.controller-manager.resources }} + resources: {{- toYaml .Values.controller-manager.resources | nindent 12 }} + {{- end }} + volumeMounts: + - mountPath: /etc/localtime + name: host-time + readOnly: true + {{- if .Values.controller-manager.webhook.enabled }} + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + {{- end }} + {{- if .Values.controller-manager.extraVolumeMounts }} + {{- include "common.tplvalues.render" (dict "value" .Values.controller-manager.extraVolumeMounts "context" $) | nindent 12 }} + {{- end }} + volumes: + - hostPath: + path: /etc/localtime + type: "" + name: host-time + {{- if .Values.controller-manager.webhook.enabled }} + - name: cert + secret: + defaultMode: 420 + secretName: kubekey-webhook-service-cert + {{- end }} + {{- if .Values.controller-manager.extraVolumes }} + {{- include "common.tplvalues.render" (dict "value" .Values.controller-manager.extraVolumes "context" $) | nindent 8 }} + {{- end }} diff --git a/config/kubekey/templates/role.yaml b/config/kubekey/templates/role.yaml new file mode 100644 index 000000000..5f0b53b76 --- /dev/null +++ b/config/kubekey/templates/role.yaml @@ -0,0 +1,57 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubekey +rules: +- apiGroups: + - "" + resources: + - events + - pods + - serviceaccounts + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubekey.kubesphere.io + resources: + - '*' + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterrolebindings + - clusterroles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch diff --git a/config/kubekey/templates/serviceaccount.yaml b/config/kubekey/templates/serviceaccount.yaml new file mode 100644 index 000000000..d67d99295 --- /dev/null +++ b/config/kubekey/templates/serviceaccount.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubekey + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubekey +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubekey +subjects: + - kind: ServiceAccount + name: kubekey diff --git a/config/kubekey/templates/webhook.yaml b/config/kubekey/templates/webhook.yaml new file mode 100644 index 000000000..a98f8c4ba --- /dev/null +++ b/config/kubekey/templates/webhook.yaml @@ -0,0 +1,88 @@ +{{- if .Values.controller-manager.webhook.enabled }} +{{- $isCertManager := or (.Capabilities.APIVersions.Has "cert-manager.io/v1beta1") (.Capabilities.APIVersions.Has "cert-manager.io/v1alpha2") (.Capabilities.APIVersions.Has "certmanager.k8s.io/v1alpha1") (.Capabilities.APIVersions.Has "cert-manager.io/v1") }} +{{- $certManagerApiVersion := "cert-manager.io/v1" }} +{{- if or (.Capabilities.APIVersions.Has "cert-manager.io/v1") }} +{{- $certManagerApiVersion = "cert-manager.io/v1" }} +{{- else if or (.Capabilities.APIVersions.Has "cert-manager.io/v1beta1") }} +{{- $certManagerApiVersion = "cert-manager.io/v1beta1" }} +{{- else if or (.Capabilities.APIVersions.Has "cert-manager.io/v1alpha2") }} +{{- $certManagerApiVersion = "cert-manager.io/v1alpha2" }} +{{- else if or (.Capabilities.APIVersions.Has "certmanager.k8s.io/v1alpha1") }} +{{- $certManagerApiVersion = "cert-manager.io/v1alpha1" }} +{{- end }} +{{- $ca := genCA "self-signed-ca" 3650 }} +{{- $cn := printf "%s-extensions-museum" .Release.Name }} +{{- $altName1 := printf "kubekey-webhook-service.%s" .Release.Namespace }} +{{- $altName2 := printf "kubekey-webhook-service.%s.svc" .Release.Namespace }} +{{- $cert := genSignedCert $cn nil (list $altName1 $altName2) 3650 $ca }} +{{- if $isCertManager }} +--- +apiVersion: {{ $certManagerApiVersion }} +kind: Issuer +metadata: + name: kubekey-selfsigned-issuer + namespace: {{ .Release.Namespace }} +spec: + selfSigned: {} +--- +apiVersion: {{ $certManagerApiVersion }} +kind: Certificate +metadata: + name: kubekey-serving-cert + namespace: {{ .Release.Namespace }} +spec: + dnsNames: + - kubekey-webhook-service.{{ .Release.Namespace }} + - kubekey-webhook-service.{{ .Release.Namespace }}.svc + issuerRef: + kind: Issuer + name: kubekey-selfsigned-issuer + secretName: kubekey-webhook-service-cert +{{- else }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: kubekey-webhook-service-cert + namespace: {{ .Release.Namespace }} +type: kubernetes.io/tls +data: + ca.crt: {{ b64enc $ca.Cert }} + tls.crt: {{ b64enc $cert.Cert }} + tls.key: {{ b64enc $cert.Key }} +{{- end }} +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: default-kubekey + namespace: {{ .Release.Namespace }} + {{- if $isCertManager }} + annotations: + cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/kubekey-serving-cert + {{- end }} +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kk-webhook-service + namespace: {{ .Release.Namespace }} + path: /mutate-kubekey-kubesphere-io-v1beta1-playbook + {{- if not $isCertManager }} + caBundle: {{ b64enc $ca.Cert }} + {{- end }} + failurePolicy: Fail + name: default.playbook.kubekey.kubesphere.io + rules: + - apiGroups: + - kubekey.kubesphere.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - playbooks + sideEffects: None +{{- end }} \ No newline at end of file diff --git a/config/kubekey/values.yaml b/config/kubekey/values.yaml new file mode 100644 index 000000000..ee8479695 --- /dev/null +++ b/config/kubekey/values.yaml @@ -0,0 +1,80 @@ +## @section Common parameters +## +global: + imageRegistry: "docker.io" + tag: "" + imagePullSecrets: [] + +controller-manager: + webhook: + enabled: true + # tolerations of operator pod + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + key: node.kubernetes.io/not-ready + operator: Exists + tolerationSeconds: 60 + - effect: NoExecute + key: node.kubernetes.io/unreachable + operator: Exists + tolerationSeconds: 60 + # affinity of operator pod + affinity: { } + # nodeSelector of operator pod + nodeSelector: { } + # dnsPolicy of operator pod + dnsPolicy: Default + # restartPolicy of operator pod + restartPolicy: Always + # schedulerName of operator pod + schedulerName: default-scheduler + # terminationGracePeriodSeconds of operator pod + terminationGracePeriodSeconds: 30 + # replica of operator deployment + replicaCount: 1 + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## e.g: + ## pullSecrets: + ## - myRegistryKeySecretName + pullSecrets: [] + image: + registry: "" + repository: kubesphere/kk-controller-manager + tag: "" + digest: "" + pullPolicy: IfNotPresent + ## + ## @param resources.limits The resources limits for the operator containers + ## @param resources.requests The requested resources for the operator containers + ## + resources: + limits: + cpu: 1 + memory: 1000Mi + requests: + cpu: 30m + memory: 50Mi + command: [] + ## @param extraEnvVars Array with extra environment variables to add to haproxy nodes + ## + extraEnvVars: [] + ## @param extraVolumeMounts Optionally specify extra list of additional volumeMounts for the haproxy container(s) + ## + extraVolumeMounts: [] + ## @param extraVolumes Optionally specify extra list of additional volumes for the haproxy pod(s) + ## + extraVolumes: [] + +executor: + image: + registry: "" + repository: kubesphere/kk-executor + tag: "" + digest: "" + pullPolicy: IfNotPresent diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml deleted file mode 100644 index 7394a6d05..000000000 --- a/config/manager/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: - - manager.yaml diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml deleted file mode 100644 index 36967c890..000000000 --- a/config/manager/manager.yaml +++ /dev/null @@ -1,53 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - labels: - control-plane: controller-manager - name: system ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - control-plane: controller-manager -spec: - selector: - matchLabels: - control-plane: controller-manager - replicas: 1 - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: manager - labels: - control-plane: controller-manager - spec: - containers: - - args: - - "--leader-elect" - - "--v=4" - image: controller:latest - name: manager - securityContext: - allowPrivilegeEscalation: false - livenessProbe: - httpGet: - path: /healthz - port: 9440 - initialDelaySeconds: 15 - periodSeconds: 20 - readinessProbe: - httpGet: - path: /readyz - port: 9440 - initialDelaySeconds: 5 - periodSeconds: 10 - serviceAccountName: controller-manager - terminationGracePeriodSeconds: 10 - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane diff --git a/config/manifests/kustomization.yaml b/config/manifests/kustomization.yaml deleted file mode 100644 index d2a86b1af..000000000 --- a/config/manifests/kustomization.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# These resources constitute the fully configured set of manifests -# used to generate the 'manifests/' directory in a bundle. -resources: -- bases/cluster-api-kubekey-provider.clusterserviceversion.yaml -- ../default -- ../samples -- ../scorecard - -# [WEBHOOK] To enable webhooks, uncomment all the sections with [WEBHOOK] prefix. -# Do NOT uncomment sections with prefix [CERTMANAGER], as OLM does not support cert-manager. -# These patches remove the unnecessary "cert" volume and its manager container volumeMount. -#patchesJson6902: -#- target: -# group: apps -# version: v1 -# kind: Deployment -# name: controller-manager -# namespace: system -# patch: |- -# # Remove the manager container's "cert" volumeMount, since OLM will create and mount a set of certs. -# # Update the indices in this path if adding or removing containers/volumeMounts in the manager's Deployment. -# - op: remove -# path: /spec/template/spec/containers/1/volumeMounts/0 -# # Remove the "cert" volume, since OLM will create and mount a set of certs. -# # Update the indices in this path if adding or removing volumes in the manager's Deployment. -# - op: remove -# path: /spec/template/spec/volumes/0 diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml deleted file mode 100644 index ed137168a..000000000 --- a/config/prometheus/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- monitor.yaml diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml deleted file mode 100644 index d19136ae7..000000000 --- a/config/prometheus/monitor.yaml +++ /dev/null @@ -1,20 +0,0 @@ - -# Prometheus Monitor Service (Metrics) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - labels: - control-plane: controller-manager - name: controller-manager-metrics-monitor - namespace: system -spec: - endpoints: - - path: /metrics - port: https - scheme: https - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - tlsConfig: - insecureSkipVerify: true - selector: - matchLabels: - control-plane: controller-manager diff --git a/config/rbac/auth_proxy_client_clusterrole.yaml b/config/rbac/auth_proxy_client_clusterrole.yaml deleted file mode 100644 index 51a75db47..000000000 --- a/config/rbac/auth_proxy_client_clusterrole.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: metrics-reader -rules: -- nonResourceURLs: - - "/metrics" - verbs: - - get diff --git a/config/rbac/auth_proxy_role.yaml b/config/rbac/auth_proxy_role.yaml deleted file mode 100644 index 80e1857c5..000000000 --- a/config/rbac/auth_proxy_role.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create diff --git a/config/rbac/auth_proxy_role_binding.yaml b/config/rbac/auth_proxy_role_binding.yaml deleted file mode 100644 index ec7acc0a1..000000000 --- a/config/rbac/auth_proxy_role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: proxy-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml deleted file mode 100644 index 71f179727..000000000 --- a/config/rbac/auth_proxy_service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - control-plane: controller-manager - name: controller-manager-metrics-service - namespace: system -spec: - ports: - - name: https - port: 8443 - protocol: TCP - targetPort: https - selector: - control-plane: controller-manager diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml deleted file mode 100644 index 731832a6a..000000000 --- a/config/rbac/kustomization.yaml +++ /dev/null @@ -1,18 +0,0 @@ -resources: -# All RBAC will be applied under this service account in -# the deployment namespace. You may comment out this resource -# if your manager will use a service account that exists at -# runtime. Be sure to update RoleBinding and ClusterRoleBinding -# subjects if changing service account names. -- service_account.yaml -- role.yaml -- role_binding.yaml -- leader_election_role.yaml -- leader_election_role_binding.yaml -# Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) -# which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml deleted file mode 100644 index 4190ec805..000000000 --- a/config/rbac/leader_election_role.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# permissions to do leader election. -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: leader-election-role -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch diff --git a/config/rbac/leader_election_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml deleted file mode 100644 index 1d1321ed4..000000000 --- a/config/rbac/leader_election_role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: leader-election-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: leader-election-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml deleted file mode 100644 index 432983034..000000000 --- a/config/rbac/role.yaml +++ /dev/null @@ -1,162 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: manager-role -rules: -- apiGroups: - - "" - resources: - - configmaps - - events - - secrets - verbs: - - create - - get - - list - - patch - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch -- apiGroups: - - cluster.x-k8s.io - resources: - - '*' - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - cluster.x-k8s.io - resources: - - clusters - - clusters/status - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - cluster.x-k8s.io - resources: - - machinedeployments - - machinedeployments/status - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - cluster.x-k8s.io - resources: - - machines - - machines/status - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - cluster.x-k8s.io - resources: - - machinesets - - machinesets/status - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - controlplane.cluster.x-k8s.io - resources: - - '*' - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - infrastructure.cluster.x-k8s.io - resources: - - '*' - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - infrastructure.cluster.x-k8s.io - resources: - - kkinstances - - kkinstances/finalizers - - kkinstances/status - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - infrastructure.cluster.x-k8s.io - resources: - - kkmachines - - kkmachines/finalizers - - kkmachines/status - verbs: - - create - - delete - - get - - list - - patch - - update - - watch diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml deleted file mode 100644 index 2070ede44..000000000 --- a/config/rbac/role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: manager-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml deleted file mode 100644 index 7cd6025bf..000000000 --- a/config/rbac/service_account.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: controller-manager - namespace: system diff --git a/config/samples/hello-kubekey/hello.yaml b/config/samples/hello-kubekey/hello.yaml deleted file mode 100644 index b2b584174..000000000 --- a/config/samples/hello-kubekey/hello.yaml +++ /dev/null @@ -1,188 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: quick-start -spec: - clusterNetwork: - services: - cidrBlocks: [ "10.233.0.0/18" ] - pods: - cidrBlocks: [ "10.233.64.0/18" ] - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: KKCluster - name: quick-start - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: quick-start-control-plane ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: KKCluster -metadata: - name: quick-start -spec: - component: - zone: "" - nodes: - auth: - user: ubuntu - password: Qcloud@123 - instances: - - name: test1 - address: 192.168.0.3 - - name: test2 - address: 192.168.0.4 - - name: test3 - address: 192.168.0.5 - controlPlaneLoadBalancer: - host: 192.168.0.100 ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: quick-start-control-plane -spec: - replicas: 1 - version: v1.22.10 - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: KKMachineTemplate - name: quick-start - kubeadmConfigSpec: - files: - - content: | - apiVersion: v1 - kind: Pod - metadata: - creationTimestamp: null - name: kube-vip - namespace: kube-system - spec: - containers: - - args: - - manager - env: - - name: address - value: 192.168.0.100 - - name: vip_interface - value: ${VIP_NETWORK_INTERFACE=""} - - name: vip_arp - value: "true" - - name: port - value: "6443" - - name: vip_cidr - value: "32" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: vip_ddns - value: "false" - - name: svc_enable - value: "true" - - name: vip_leaderelection - value: "true" - - name: vip_leaseduration - value: "5" - - name: vip_renewdeadline - value: "3" - - name: vip_retryperiod - value: "1" - - name: lb_enable - value: "true" - - name: lb_port - value: "6443" - image: ghcr.io/kube-vip/kube-vip:v0.5.0 - imagePullPolicy: IfNotPresent - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - volumeMounts: - - mountPath: /etc/kubernetes/admin.conf - name: kubeconfig - hostNetwork: true - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 - volumes: - - hostPath: - path: /etc/kubernetes/admin.conf - type: FileOrCreate - name: kubeconfig - status: {} - owner: root:root - path: /etc/kubernetes/manifests/kube-vip.yaml - initConfiguration: - nodeRegistration: - criSocket: unix:///var/run/containerd/containerd.sock - joinConfiguration: - nodeRegistration: - criSocket: unix:///var/run/containerd/containerd.sock ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: KKMachineTemplate -metadata: - name: quick-start - namespace: default -spec: - template: - spec: - roles: - - control-plane - repository: - iso: "none" - update: false ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: "quick-start-md-0" -spec: - clusterName: "quick-start" - replicas: 1 - selector: - matchLabels: - template: - spec: - clusterName: "quick-start" - version: v1.22.10 - bootstrap: - configRef: - name: "quick-start-md-0" - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - infrastructureRef: - name: "quick-start-md-0" - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: KKMachineTemplate ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: KKMachineTemplate -metadata: - name: 'quick-start-md-0' -spec: - template: - spec: - roles: - - worker - repository: - iso: "none" - update: false ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: "quick-start-md-0" -spec: - template: - spec: - joinConfiguration: - nodeRegistration: - criSocket: unix:///var/run/containerd/containerd.sock \ No newline at end of file diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml deleted file mode 100644 index 2bc04a19d..000000000 --- a/config/samples/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -## Append samples you want in your CSV to this file as resources ## -resources: -- infrastructure_v1beta1_kkcluster.yaml -- infrastructure_v1beta1_kkclustertemplate.yaml -- infrastructure_v1beta1_kkmachine.yaml -- infrastructure_v1beta1_kkmachinetemplate.yaml -- infrastructure_v1beta1_kkinstance.yaml -#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/scorecard/bases/config.yaml b/config/scorecard/bases/config.yaml deleted file mode 100644 index c77047841..000000000 --- a/config/scorecard/bases/config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: scorecard.operatorframework.io/v1alpha3 -kind: Configuration -metadata: - name: config -stages: -- parallel: true - tests: [] diff --git a/config/scorecard/kustomization.yaml b/config/scorecard/kustomization.yaml deleted file mode 100644 index 50cd2d084..000000000 --- a/config/scorecard/kustomization.yaml +++ /dev/null @@ -1,16 +0,0 @@ -resources: -- bases/config.yaml -patchesJson6902: -- path: patches/basic.config.yaml - target: - group: scorecard.operatorframework.io - version: v1alpha3 - kind: Configuration - name: config -- path: patches/olm.config.yaml - target: - group: scorecard.operatorframework.io - version: v1alpha3 - kind: Configuration - name: config -#+kubebuilder:scaffold:patchesJson6902 diff --git a/config/scorecard/patches/basic.config.yaml b/config/scorecard/patches/basic.config.yaml deleted file mode 100644 index 7b4653e8d..000000000 --- a/config/scorecard/patches/basic.config.yaml +++ /dev/null @@ -1,10 +0,0 @@ -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - basic-check-spec - image: quay.io/operator-framework/scorecard-test:v1.20.0 - labels: - suite: basic - test: basic-check-spec-test diff --git a/config/scorecard/patches/olm.config.yaml b/config/scorecard/patches/olm.config.yaml deleted file mode 100644 index 39390f09c..000000000 --- a/config/scorecard/patches/olm.config.yaml +++ /dev/null @@ -1,50 +0,0 @@ -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-bundle-validation - image: quay.io/operator-framework/scorecard-test:v1.20.0 - labels: - suite: olm - test: olm-bundle-validation-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-crds-have-validation - image: quay.io/operator-framework/scorecard-test:v1.20.0 - labels: - suite: olm - test: olm-crds-have-validation-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-crds-have-resources - image: quay.io/operator-framework/scorecard-test:v1.20.0 - labels: - suite: olm - test: olm-crds-have-resources-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-spec-descriptors - image: quay.io/operator-framework/scorecard-test:v1.20.0 - labels: - suite: olm - test: olm-spec-descriptors-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-status-descriptors - image: quay.io/operator-framework/scorecard-test:v1.20.0 - labels: - suite: olm - test: olm-status-descriptors-test diff --git a/config/swagger-ui/favicon-16x16.png b/config/swagger-ui/favicon-16x16.png new file mode 100644 index 000000000..8b194e617 Binary files /dev/null and b/config/swagger-ui/favicon-16x16.png differ diff --git a/config/swagger-ui/favicon-32x32.png b/config/swagger-ui/favicon-32x32.png new file mode 100644 index 000000000..249737fe4 Binary files /dev/null and b/config/swagger-ui/favicon-32x32.png differ diff --git a/config/swagger-ui/index.css b/config/swagger-ui/index.css new file mode 100644 index 000000000..f2376fdaa --- /dev/null +++ b/config/swagger-ui/index.css @@ -0,0 +1,16 @@ +html { + box-sizing: border-box; + overflow: -moz-scrollbars-vertical; + overflow-y: scroll; +} + +*, +*:before, +*:after { + box-sizing: inherit; +} + +body { + margin: 0; + background: #fafafa; +} diff --git a/config/swagger-ui/index.html b/config/swagger-ui/index.html new file mode 100644 index 000000000..84ae62d3d --- /dev/null +++ b/config/swagger-ui/index.html @@ -0,0 +1,19 @@ + + + + + + Swagger UI + + + + + + + +
+ + + + + diff --git a/config/swagger-ui/oauth2-redirect.html b/config/swagger-ui/oauth2-redirect.html new file mode 100644 index 000000000..564091718 --- /dev/null +++ b/config/swagger-ui/oauth2-redirect.html @@ -0,0 +1,79 @@ + + + + Swagger UI: OAuth2 Redirect + + + + + diff --git a/config/swagger-ui/swagger-initializer.js b/config/swagger-ui/swagger-initializer.js new file mode 100644 index 000000000..446812ee7 --- /dev/null +++ b/config/swagger-ui/swagger-initializer.js @@ -0,0 +1,20 @@ +window.onload = function() { + // + + // the following lines will be replaced by docker/configurator, when it runs in a docker-container + window.ui = SwaggerUIBundle({ + url: "/apidocs.json", + dom_id: '#swagger-ui', + deepLinking: true, + presets: [ + SwaggerUIBundle.presets.apis, + SwaggerUIStandalonePreset + ], + plugins: [ + SwaggerUIBundle.plugins.DownloadUrl + ], + layout: "StandaloneLayout" + }); + + // +}; diff --git a/config/swagger-ui/swagger-ui-bundle.js b/config/swagger-ui/swagger-ui-bundle.js new file mode 100644 index 000000000..dfff077dd --- /dev/null +++ b/config/swagger-ui/swagger-ui-bundle.js @@ -0,0 +1,2 @@ +/*! For license information please see swagger-ui-bundle.js.LICENSE.txt */ +!function webpackUniversalModuleDefinition(s,o){"object"==typeof exports&&"object"==typeof module?module.exports=o():"function"==typeof define&&define.amd?define([],o):"object"==typeof exports?exports.SwaggerUIBundle=o():s.SwaggerUIBundle=o()}(this,(()=>(()=>{var s={251:(s,o)=>{o.read=function(s,o,i,a,u){var _,w,x=8*u-a-1,C=(1<>1,L=-7,B=i?u-1:0,$=i?-1:1,V=s[o+B];for(B+=$,_=V&(1<<-L)-1,V>>=-L,L+=x;L>0;_=256*_+s[o+B],B+=$,L-=8);for(w=_&(1<<-L)-1,_>>=-L,L+=a;L>0;w=256*w+s[o+B],B+=$,L-=8);if(0===_)_=1-j;else{if(_===C)return w?NaN:1/0*(V?-1:1);w+=Math.pow(2,a),_-=j}return(V?-1:1)*w*Math.pow(2,_-a)},o.write=function(s,o,i,a,u,_){var w,x,C,j=8*_-u-1,L=(1<>1,$=23===u?Math.pow(2,-24)-Math.pow(2,-77):0,V=a?0:_-1,U=a?1:-1,z=o<0||0===o&&1/o<0?1:0;for(o=Math.abs(o),isNaN(o)||o===1/0?(x=isNaN(o)?1:0,w=L):(w=Math.floor(Math.log(o)/Math.LN2),o*(C=Math.pow(2,-w))<1&&(w--,C*=2),(o+=w+B>=1?$/C:$*Math.pow(2,1-B))*C>=2&&(w++,C/=2),w+B>=L?(x=0,w=L):w+B>=1?(x=(o*C-1)*Math.pow(2,u),w+=B):(x=o*Math.pow(2,B-1)*Math.pow(2,u),w=0));u>=8;s[i+V]=255&x,V+=U,x/=256,u-=8);for(w=w<0;s[i+V]=255&w,V+=U,w/=256,j-=8);s[i+V-U]|=128*z}},462:(s,o,i)=>{"use strict";var a=i(40975);s.exports=a},659:(s,o,i)=>{var a=i(51873),u=Object.prototype,_=u.hasOwnProperty,w=u.toString,x=a?a.toStringTag:void 0;s.exports=function getRawTag(s){var o=_.call(s,x),i=s[x];try{s[x]=void 0;var a=!0}catch(s){}var u=w.call(s);return a&&(o?s[x]=i:delete s[x]),u}},694:(s,o,i)=>{"use strict";i(91599);var a=i(37257);i(12560),s.exports=a},953:(s,o,i)=>{"use strict";s.exports=i(53375)},1733:s=>{var o=/[^\x00-\x2f\x3a-\x40\x5b-\x60\x7b-\x7f]+/g;s.exports=function asciiWords(s){return s.match(o)||[]}},1882:(s,o,i)=>{var a=i(72552),u=i(23805);s.exports=function isFunction(s){if(!u(s))return!1;var o=a(s);return"[object Function]"==o||"[object GeneratorFunction]"==o||"[object AsyncFunction]"==o||"[object Proxy]"==o}},1907:(s,o,i)=>{"use strict";var a=i(41505),u=Function.prototype,_=u.call,w=a&&u.bind.bind(_,_);s.exports=a?w:function(s){return function(){return _.apply(s,arguments)}}},2205:function(s,o,i){var a;a=void 0!==i.g?i.g:this,s.exports=function(s){if(s.CSS&&s.CSS.escape)return s.CSS.escape;var cssEscape=function(s){if(0==arguments.length)throw new TypeError("`CSS.escape` requires an argument.");for(var o,i=String(s),a=i.length,u=-1,_="",w=i.charCodeAt(0);++u=1&&o<=31||127==o||0==u&&o>=48&&o<=57||1==u&&o>=48&&o<=57&&45==w?"\\"+o.toString(16)+" ":0==u&&1==a&&45==o||!(o>=128||45==o||95==o||o>=48&&o<=57||o>=65&&o<=90||o>=97&&o<=122)?"\\"+i.charAt(u):i.charAt(u):_+="�";return _};return s.CSS||(s.CSS={}),s.CSS.escape=cssEscape,cssEscape}(a)},2209:(s,o,i)=>{"use strict";var a,u=i(9404),_=function productionTypeChecker(){invariant(!1,"ImmutablePropTypes type checking code is stripped in production.")};_.isRequired=_;var w=function getProductionTypeChecker(){return _};function getPropType(s){var o=typeof s;return Array.isArray(s)?"array":s instanceof RegExp?"object":s instanceof u.Iterable?"Immutable."+s.toSource().split(" ")[0]:o}function createChainableTypeChecker(s){function checkType(o,i,a,u,_,w){for(var x=arguments.length,C=Array(x>6?x-6:0),j=6;j>",null!=i[a]?s.apply(void 0,[i,a,u,_,w].concat(C)):o?new Error("Required "+_+" `"+w+"` was not specified in `"+u+"`."):void 0}var o=checkType.bind(null,!1);return o.isRequired=checkType.bind(null,!0),o}function createIterableSubclassTypeChecker(s,o){return function createImmutableTypeChecker(s,o){return createChainableTypeChecker((function validate(i,a,u,_,w){var x=i[a];if(!o(x)){var C=getPropType(x);return new Error("Invalid "+_+" `"+w+"` of type `"+C+"` supplied to `"+u+"`, expected `"+s+"`.")}return null}))}("Iterable."+s,(function(s){return u.Iterable.isIterable(s)&&o(s)}))}(a={listOf:w,mapOf:w,orderedMapOf:w,setOf:w,orderedSetOf:w,stackOf:w,iterableOf:w,recordOf:w,shape:w,contains:w,mapContains:w,orderedMapContains:w,list:_,map:_,orderedMap:_,set:_,orderedSet:_,stack:_,seq:_,record:_,iterable:_}).iterable.indexed=createIterableSubclassTypeChecker("Indexed",u.Iterable.isIndexed),a.iterable.keyed=createIterableSubclassTypeChecker("Keyed",u.Iterable.isKeyed),s.exports=a},2404:(s,o,i)=>{var a=i(60270);s.exports=function isEqual(s,o){return a(s,o)}},2523:s=>{s.exports=function baseFindIndex(s,o,i,a){for(var u=s.length,_=i+(a?1:-1);a?_--:++_{"use strict";var a=i(45951),u=Object.defineProperty;s.exports=function(s,o){try{u(a,s,{value:o,configurable:!0,writable:!0})}catch(i){a[s]=o}return o}},2694:(s,o,i)=>{"use strict";var a=i(6925);function emptyFunction(){}function emptyFunctionWithReset(){}emptyFunctionWithReset.resetWarningCache=emptyFunction,s.exports=function(){function shim(s,o,i,u,_,w){if(w!==a){var x=new Error("Calling PropTypes validators directly is not supported by the `prop-types` package. Use PropTypes.checkPropTypes() to call them. Read more at http://fb.me/use-check-prop-types");throw x.name="Invariant Violation",x}}function getShim(){return shim}shim.isRequired=shim;var s={array:shim,bigint:shim,bool:shim,func:shim,number:shim,object:shim,string:shim,symbol:shim,any:shim,arrayOf:getShim,element:shim,elementType:shim,instanceOf:getShim,node:shim,objectOf:getShim,oneOf:getShim,oneOfType:getShim,shape:getShim,exact:getShim,checkPropTypes:emptyFunctionWithReset,resetWarningCache:emptyFunction};return s.PropTypes=s,s}},2874:s=>{s.exports={}},2875:(s,o,i)=>{"use strict";var a=i(23045),u=i(80376);s.exports=Object.keys||function keys(s){return a(s,u)}},2955:(s,o,i)=>{"use strict";var a,u=i(65606);function _defineProperty(s,o,i){return(o=function _toPropertyKey(s){var o=function _toPrimitive(s,o){if("object"!=typeof s||null===s)return s;var i=s[Symbol.toPrimitive];if(void 0!==i){var a=i.call(s,o||"default");if("object"!=typeof a)return a;throw new TypeError("@@toPrimitive must return a primitive value.")}return("string"===o?String:Number)(s)}(s,"string");return"symbol"==typeof o?o:String(o)}(o))in s?Object.defineProperty(s,o,{value:i,enumerable:!0,configurable:!0,writable:!0}):s[o]=i,s}var _=i(86238),w=Symbol("lastResolve"),x=Symbol("lastReject"),C=Symbol("error"),j=Symbol("ended"),L=Symbol("lastPromise"),B=Symbol("handlePromise"),$=Symbol("stream");function createIterResult(s,o){return{value:s,done:o}}function readAndResolve(s){var o=s[w];if(null!==o){var i=s[$].read();null!==i&&(s[L]=null,s[w]=null,s[x]=null,o(createIterResult(i,!1)))}}function onReadable(s){u.nextTick(readAndResolve,s)}var V=Object.getPrototypeOf((function(){})),U=Object.setPrototypeOf((_defineProperty(a={get stream(){return this[$]},next:function next(){var s=this,o=this[C];if(null!==o)return Promise.reject(o);if(this[j])return Promise.resolve(createIterResult(void 0,!0));if(this[$].destroyed)return new Promise((function(o,i){u.nextTick((function(){s[C]?i(s[C]):o(createIterResult(void 0,!0))}))}));var i,a=this[L];if(a)i=new Promise(function wrapForNext(s,o){return function(i,a){s.then((function(){o[j]?i(createIterResult(void 0,!0)):o[B](i,a)}),a)}}(a,this));else{var _=this[$].read();if(null!==_)return Promise.resolve(createIterResult(_,!1));i=new Promise(this[B])}return this[L]=i,i}},Symbol.asyncIterator,(function(){return this})),_defineProperty(a,"return",(function _return(){var s=this;return new Promise((function(o,i){s[$].destroy(null,(function(s){s?i(s):o(createIterResult(void 0,!0))}))}))})),a),V);s.exports=function createReadableStreamAsyncIterator(s){var o,i=Object.create(U,(_defineProperty(o={},$,{value:s,writable:!0}),_defineProperty(o,w,{value:null,writable:!0}),_defineProperty(o,x,{value:null,writable:!0}),_defineProperty(o,C,{value:null,writable:!0}),_defineProperty(o,j,{value:s._readableState.endEmitted,writable:!0}),_defineProperty(o,B,{value:function value(s,o){var a=i[$].read();a?(i[L]=null,i[w]=null,i[x]=null,s(createIterResult(a,!1))):(i[w]=s,i[x]=o)},writable:!0}),o));return i[L]=null,_(s,(function(s){if(s&&"ERR_STREAM_PREMATURE_CLOSE"!==s.code){var o=i[x];return null!==o&&(i[L]=null,i[w]=null,i[x]=null,o(s)),void(i[C]=s)}var a=i[w];null!==a&&(i[L]=null,i[w]=null,i[x]=null,a(createIterResult(void 0,!0))),i[j]=!0})),s.on("readable",onReadable.bind(null,i)),i}},3110:(s,o,i)=>{const a=i(5187),u=i(85015),_=i(98023),w=i(53812),x=i(23805),C=i(85105),j=i(86804);class Namespace{constructor(s){this.elementMap={},this.elementDetection=[],this.Element=j.Element,this.KeyValuePair=j.KeyValuePair,s&&s.noDefault||this.useDefault(),this._attributeElementKeys=[],this._attributeElementArrayKeys=[]}use(s){return s.namespace&&s.namespace({base:this}),s.load&&s.load({base:this}),this}useDefault(){return this.register("null",j.NullElement).register("string",j.StringElement).register("number",j.NumberElement).register("boolean",j.BooleanElement).register("array",j.ArrayElement).register("object",j.ObjectElement).register("member",j.MemberElement).register("ref",j.RefElement).register("link",j.LinkElement),this.detect(a,j.NullElement,!1).detect(u,j.StringElement,!1).detect(_,j.NumberElement,!1).detect(w,j.BooleanElement,!1).detect(Array.isArray,j.ArrayElement,!1).detect(x,j.ObjectElement,!1),this}register(s,o){return this._elements=void 0,this.elementMap[s]=o,this}unregister(s){return this._elements=void 0,delete this.elementMap[s],this}detect(s,o,i){return void 0===i||i?this.elementDetection.unshift([s,o]):this.elementDetection.push([s,o]),this}toElement(s){if(s instanceof this.Element)return s;let o;for(let i=0;i{const o=s[0].toUpperCase()+s.substr(1);this._elements[o]=this.elementMap[s]}))),this._elements}get serialiser(){return new C(this)}}C.prototype.Namespace=Namespace,s.exports=Namespace},3121:(s,o,i)=>{"use strict";var a=i(65482),u=Math.min;s.exports=function(s){var o=a(s);return o>0?u(o,9007199254740991):0}},3209:(s,o,i)=>{var a=i(91596),u=i(53320),_=i(36306),w="__lodash_placeholder__",x=128,C=Math.min;s.exports=function mergeData(s,o){var i=s[1],j=o[1],L=i|j,B=L<131,$=j==x&&8==i||j==x&&256==i&&s[7].length<=o[8]||384==j&&o[7].length<=o[8]&&8==i;if(!B&&!$)return s;1&j&&(s[2]=o[2],L|=1&i?0:4);var V=o[3];if(V){var U=s[3];s[3]=U?a(U,V,o[4]):V,s[4]=U?_(s[3],w):o[4]}return(V=o[5])&&(U=s[5],s[5]=U?u(U,V,o[6]):V,s[6]=U?_(s[5],w):o[6]),(V=o[7])&&(s[7]=V),j&x&&(s[8]=null==s[8]?o[8]:C(s[8],o[8])),null==s[9]&&(s[9]=o[9]),s[0]=o[0],s[1]=L,s}},3650:(s,o,i)=>{var a=i(74335)(Object.keys,Object);s.exports=a},3656:(s,o,i)=>{s=i.nmd(s);var a=i(9325),u=i(89935),_=o&&!o.nodeType&&o,w=_&&s&&!s.nodeType&&s,x=w&&w.exports===_?a.Buffer:void 0,C=(x?x.isBuffer:void 0)||u;s.exports=C},4509:(s,o,i)=>{var a=i(12651);s.exports=function mapCacheHas(s){return a(this,s).has(s)}},4640:s=>{"use strict";var o=String;s.exports=function(s){try{return o(s)}catch(s){return"Object"}}},4664:(s,o,i)=>{var a=i(79770),u=i(63345),_=Object.prototype.propertyIsEnumerable,w=Object.getOwnPropertySymbols,x=w?function(s){return null==s?[]:(s=Object(s),a(w(s),(function(o){return _.call(s,o)})))}:u;s.exports=x},4901:(s,o,i)=>{var a=i(72552),u=i(30294),_=i(40346),w={};w["[object Float32Array]"]=w["[object Float64Array]"]=w["[object Int8Array]"]=w["[object Int16Array]"]=w["[object Int32Array]"]=w["[object Uint8Array]"]=w["[object Uint8ClampedArray]"]=w["[object Uint16Array]"]=w["[object Uint32Array]"]=!0,w["[object Arguments]"]=w["[object Array]"]=w["[object ArrayBuffer]"]=w["[object Boolean]"]=w["[object DataView]"]=w["[object Date]"]=w["[object Error]"]=w["[object Function]"]=w["[object Map]"]=w["[object Number]"]=w["[object Object]"]=w["[object RegExp]"]=w["[object Set]"]=w["[object String]"]=w["[object WeakMap]"]=!1,s.exports=function baseIsTypedArray(s){return _(s)&&u(s.length)&&!!w[a(s)]}},4993:(s,o,i)=>{"use strict";var a=i(16946),u=i(74239);s.exports=function(s){return a(u(s))}},5187:s=>{s.exports=function isNull(s){return null===s}},5419:s=>{s.exports=function(s,o,i,a){var u=new Blob(void 0!==a?[a,s]:[s],{type:i||"application/octet-stream"});if(void 0!==window.navigator.msSaveBlob)window.navigator.msSaveBlob(u,o);else{var _=window.URL&&window.URL.createObjectURL?window.URL.createObjectURL(u):window.webkitURL.createObjectURL(u),w=document.createElement("a");w.style.display="none",w.href=_,w.setAttribute("download",o),void 0===w.download&&w.setAttribute("target","_blank"),document.body.appendChild(w),w.click(),setTimeout((function(){document.body.removeChild(w),window.URL.revokeObjectURL(_)}),200)}}},5556:(s,o,i)=>{s.exports=i(2694)()},5861:(s,o,i)=>{var a=i(55580),u=i(68223),_=i(32804),w=i(76545),x=i(28303),C=i(72552),j=i(47473),L="[object Map]",B="[object Promise]",$="[object Set]",V="[object WeakMap]",U="[object DataView]",z=j(a),Y=j(u),Z=j(_),ee=j(w),ie=j(x),ae=C;(a&&ae(new a(new ArrayBuffer(1)))!=U||u&&ae(new u)!=L||_&&ae(_.resolve())!=B||w&&ae(new w)!=$||x&&ae(new x)!=V)&&(ae=function(s){var o=C(s),i="[object Object]"==o?s.constructor:void 0,a=i?j(i):"";if(a)switch(a){case z:return U;case Y:return L;case Z:return B;case ee:return $;case ie:return V}return o}),s.exports=ae},6048:s=>{s.exports=function negate(s){if("function"!=typeof s)throw new TypeError("Expected a function");return function(){var o=arguments;switch(o.length){case 0:return!s.call(this);case 1:return!s.call(this,o[0]);case 2:return!s.call(this,o[0],o[1]);case 3:return!s.call(this,o[0],o[1],o[2])}return!s.apply(this,o)}}},6205:s=>{s.exports={ROOT:0,GROUP:1,POSITION:2,SET:3,RANGE:4,REPETITION:5,REFERENCE:6,CHAR:7}},6233:(s,o,i)=>{const a=i(6048),u=i(10316),_=i(92340);class ArrayElement extends u{constructor(s,o,i){super(s||[],o,i),this.element="array"}primitive(){return"array"}get(s){return this.content[s]}getValue(s){const o=this.get(s);if(o)return o.toValue()}getIndex(s){return this.content[s]}set(s,o){return this.content[s]=this.refract(o),this}remove(s){const o=this.content.splice(s,1);return o.length?o[0]:null}map(s,o){return this.content.map(s,o)}flatMap(s,o){return this.map(s,o).reduce(((s,o)=>s.concat(o)),[])}compactMap(s,o){const i=[];return this.forEach((a=>{const u=s.bind(o)(a);u&&i.push(u)})),i}filter(s,o){return new _(this.content.filter(s,o))}reject(s,o){return this.filter(a(s),o)}reduce(s,o){let i,a;void 0!==o?(i=0,a=this.refract(o)):(i=1,a="object"===this.primitive()?this.first.value:this.first);for(let o=i;o{s.bind(o)(i,this.refract(a))}))}shift(){return this.content.shift()}unshift(s){this.content.unshift(this.refract(s))}push(s){return this.content.push(this.refract(s)),this}add(s){this.push(s)}findElements(s,o){const i=o||{},a=!!i.recursive,u=void 0===i.results?[]:i.results;return this.forEach(((o,i,_)=>{a&&void 0!==o.findElements&&o.findElements(s,{results:u,recursive:a}),s(o,i,_)&&u.push(o)})),u}find(s){return new _(this.findElements(s,{recursive:!0}))}findByElement(s){return this.find((o=>o.element===s))}findByClass(s){return this.find((o=>o.classes.includes(s)))}getById(s){return this.find((o=>o.id.toValue()===s)).first}includes(s){return this.content.some((o=>o.equals(s)))}contains(s){return this.includes(s)}empty(){return new this.constructor([])}"fantasy-land/empty"(){return this.empty()}concat(s){return new this.constructor(this.content.concat(s.content))}"fantasy-land/concat"(s){return this.concat(s)}"fantasy-land/map"(s){return new this.constructor(this.map(s))}"fantasy-land/chain"(s){return this.map((o=>s(o)),this).reduce(((s,o)=>s.concat(o)),this.empty())}"fantasy-land/filter"(s){return new this.constructor(this.content.filter(s))}"fantasy-land/reduce"(s,o){return this.content.reduce(s,o)}get length(){return this.content.length}get isEmpty(){return 0===this.content.length}get first(){return this.getIndex(0)}get second(){return this.getIndex(1)}get last(){return this.getIndex(this.length-1)}}ArrayElement.empty=function empty(){return new this},ArrayElement["fantasy-land/empty"]=ArrayElement.empty,"undefined"!=typeof Symbol&&(ArrayElement.prototype[Symbol.iterator]=function symbol(){return this.content[Symbol.iterator]()}),s.exports=ArrayElement},6499:(s,o,i)=>{"use strict";var a=i(1907),u=0,_=Math.random(),w=a(1..toString);s.exports=function(s){return"Symbol("+(void 0===s?"":s)+")_"+w(++u+_,36)}},6925:s=>{"use strict";s.exports="SECRET_DO_NOT_PASS_THIS_OR_YOU_WILL_BE_FIRED"},7057:(s,o,i)=>{"use strict";var a=i(11470).charAt,u=i(90160),_=i(64932),w=i(60183),x=i(59550),C="String Iterator",j=_.set,L=_.getterFor(C);w(String,"String",(function(s){j(this,{type:C,string:u(s),index:0})}),(function next(){var s,o=L(this),i=o.string,u=o.index;return u>=i.length?x(void 0,!0):(s=a(i,u),o.index+=s.length,x(s,!1))}))},7309:(s,o,i)=>{var a=i(62006)(i(24713));s.exports=a},7376:s=>{"use strict";s.exports=!0},7463:(s,o,i)=>{"use strict";var a=i(98828),u=i(62250),_=/#|\.prototype\./,isForced=function(s,o){var i=x[w(s)];return i===j||i!==C&&(u(o)?a(o):!!o)},w=isForced.normalize=function(s){return String(s).replace(_,".").toLowerCase()},x=isForced.data={},C=isForced.NATIVE="N",j=isForced.POLYFILL="P";s.exports=isForced},7666:(s,o,i)=>{var a=i(84851),u=i(953);function _extends(){var o;return s.exports=_extends=a?u(o=a).call(o):function(s){for(var o=1;o{const a=i(6205);o.wordBoundary=()=>({type:a.POSITION,value:"b"}),o.nonWordBoundary=()=>({type:a.POSITION,value:"B"}),o.begin=()=>({type:a.POSITION,value:"^"}),o.end=()=>({type:a.POSITION,value:"$"})},8068:s=>{"use strict";var o=(()=>{var s=Object.defineProperty,o=Object.getOwnPropertyDescriptor,i=Object.getOwnPropertyNames,a=Object.getOwnPropertySymbols,u=Object.prototype.hasOwnProperty,_=Object.prototype.propertyIsEnumerable,__defNormalProp=(o,i,a)=>i in o?s(o,i,{enumerable:!0,configurable:!0,writable:!0,value:a}):o[i]=a,__spreadValues=(s,o)=>{for(var i in o||(o={}))u.call(o,i)&&__defNormalProp(s,i,o[i]);if(a)for(var i of a(o))_.call(o,i)&&__defNormalProp(s,i,o[i]);return s},__publicField=(s,o,i)=>__defNormalProp(s,"symbol"!=typeof o?o+"":o,i),w={};((o,i)=>{for(var a in i)s(o,a,{get:i[a],enumerable:!0})})(w,{DEFAULT_OPTIONS:()=>C,DEFAULT_UUID_LENGTH:()=>x,default:()=>B});var x=6,C={dictionary:"alphanum",shuffle:!0,debug:!1,length:x,counter:0},j=class _ShortUniqueId{constructor(s={}){__publicField(this,"counter"),__publicField(this,"debug"),__publicField(this,"dict"),__publicField(this,"version"),__publicField(this,"dictIndex",0),__publicField(this,"dictRange",[]),__publicField(this,"lowerBound",0),__publicField(this,"upperBound",0),__publicField(this,"dictLength",0),__publicField(this,"uuidLength"),__publicField(this,"_digit_first_ascii",48),__publicField(this,"_digit_last_ascii",58),__publicField(this,"_alpha_lower_first_ascii",97),__publicField(this,"_alpha_lower_last_ascii",123),__publicField(this,"_hex_last_ascii",103),__publicField(this,"_alpha_upper_first_ascii",65),__publicField(this,"_alpha_upper_last_ascii",91),__publicField(this,"_number_dict_ranges",{digits:[this._digit_first_ascii,this._digit_last_ascii]}),__publicField(this,"_alpha_dict_ranges",{lowerCase:[this._alpha_lower_first_ascii,this._alpha_lower_last_ascii],upperCase:[this._alpha_upper_first_ascii,this._alpha_upper_last_ascii]}),__publicField(this,"_alpha_lower_dict_ranges",{lowerCase:[this._alpha_lower_first_ascii,this._alpha_lower_last_ascii]}),__publicField(this,"_alpha_upper_dict_ranges",{upperCase:[this._alpha_upper_first_ascii,this._alpha_upper_last_ascii]}),__publicField(this,"_alphanum_dict_ranges",{digits:[this._digit_first_ascii,this._digit_last_ascii],lowerCase:[this._alpha_lower_first_ascii,this._alpha_lower_last_ascii],upperCase:[this._alpha_upper_first_ascii,this._alpha_upper_last_ascii]}),__publicField(this,"_alphanum_lower_dict_ranges",{digits:[this._digit_first_ascii,this._digit_last_ascii],lowerCase:[this._alpha_lower_first_ascii,this._alpha_lower_last_ascii]}),__publicField(this,"_alphanum_upper_dict_ranges",{digits:[this._digit_first_ascii,this._digit_last_ascii],upperCase:[this._alpha_upper_first_ascii,this._alpha_upper_last_ascii]}),__publicField(this,"_hex_dict_ranges",{decDigits:[this._digit_first_ascii,this._digit_last_ascii],alphaDigits:[this._alpha_lower_first_ascii,this._hex_last_ascii]}),__publicField(this,"_dict_ranges",{_number_dict_ranges:this._number_dict_ranges,_alpha_dict_ranges:this._alpha_dict_ranges,_alpha_lower_dict_ranges:this._alpha_lower_dict_ranges,_alpha_upper_dict_ranges:this._alpha_upper_dict_ranges,_alphanum_dict_ranges:this._alphanum_dict_ranges,_alphanum_lower_dict_ranges:this._alphanum_lower_dict_ranges,_alphanum_upper_dict_ranges:this._alphanum_upper_dict_ranges,_hex_dict_ranges:this._hex_dict_ranges}),__publicField(this,"log",((...s)=>{const o=[...s];o[0]="[short-unique-id] ".concat(s[0]),!0!==this.debug||"undefined"==typeof console||null===console||console.log(...o)})),__publicField(this,"_normalizeDictionary",((s,o)=>{let i;if(s&&Array.isArray(s)&&s.length>1)i=s;else{i=[],this.dictIndex=0;const o="_".concat(s,"_dict_ranges"),a=this._dict_ranges[o];let u=0;for(const[,s]of Object.entries(a)){const[o,i]=s;u+=Math.abs(i-o)}i=new Array(u);let _=0;for(const[,s]of Object.entries(a)){this.dictRange=s,this.lowerBound=this.dictRange[0],this.upperBound=this.dictRange[1];const o=this.lowerBound<=this.upperBound,a=this.lowerBound,u=this.upperBound;if(o)for(let s=a;su;s--)i[_++]=String.fromCharCode(s),this.dictIndex=s}i.length=_}if(o){for(let s=i.length-1;s>0;s--){const o=Math.floor(Math.random()*(s+1));[i[s],i[o]]=[i[o],i[s]]}}return i})),__publicField(this,"setDictionary",((s,o)=>{this.dict=this._normalizeDictionary(s,o),this.dictLength=this.dict.length,this.setCounter(0)})),__publicField(this,"seq",(()=>this.sequentialUUID())),__publicField(this,"sequentialUUID",(()=>{const s=this.dictLength,o=this.dict;let i=this.counter;const a=[];do{const u=i%s;i=Math.trunc(i/s),a.push(o[u])}while(0!==i);const u=a.join("");return this.counter+=1,u})),__publicField(this,"rnd",((s=this.uuidLength||x)=>this.randomUUID(s))),__publicField(this,"randomUUID",((s=this.uuidLength||x)=>{if(null==s||s<1)throw new Error("Invalid UUID Length Provided");const o=new Array(s),i=this.dictLength,a=this.dict;for(let u=0;uthis.formattedUUID(s,o))),__publicField(this,"formattedUUID",((s,o)=>{const i={$r:this.randomUUID,$s:this.sequentialUUID,$t:this.stamp};return s.replace(/\$[rs]\d{0,}|\$t0|\$t[1-9]\d{1,}/g,(s=>{const a=s.slice(0,2),u=Number.parseInt(s.slice(2),10);return"$s"===a?i[a]().padStart(u,"0"):"$t"===a&&o?i[a](u,o):i[a](u)}))})),__publicField(this,"availableUUIDs",((s=this.uuidLength)=>Number.parseFloat(([...new Set(this.dict)].length**s).toFixed(0)))),__publicField(this,"_collisionCache",new Map),__publicField(this,"approxMaxBeforeCollision",((s=this.availableUUIDs(this.uuidLength))=>{const o=s,i=this._collisionCache.get(o);if(void 0!==i)return i;const a=Number.parseFloat(Math.sqrt(Math.PI/2*s).toFixed(20));return this._collisionCache.set(o,a),a})),__publicField(this,"collisionProbability",((s=this.availableUUIDs(this.uuidLength),o=this.uuidLength)=>Number.parseFloat((this.approxMaxBeforeCollision(s)/this.availableUUIDs(o)).toFixed(20)))),__publicField(this,"uniqueness",((s=this.availableUUIDs(this.uuidLength))=>{const o=Number.parseFloat((1-this.approxMaxBeforeCollision(s)/s).toFixed(20));return o>1?1:o<0?0:o})),__publicField(this,"getVersion",(()=>this.version)),__publicField(this,"stamp",((s,o)=>{const i=Math.floor(+(o||new Date)/1e3).toString(16);if("number"==typeof s&&0===s)return i;if("number"!=typeof s||s<10)throw new Error(["Param finalLength must be a number greater than or equal to 10,","or 0 if you want the raw hexadecimal timestamp"].join("\n"));const a=s-9,u=Math.round(Math.random()*(a>15?15:a)),_=this.randomUUID(a);return"".concat(_.substring(0,u)).concat(i).concat(_.substring(u)).concat(u.toString(16))})),__publicField(this,"parseStamp",((s,o)=>{if(o&&!/t0|t[1-9]\d{1,}/.test(o))throw new Error("Cannot extract date from a formated UUID with no timestamp in the format");const i=o?o.replace(/\$[rs]\d{0,}|\$t0|\$t[1-9]\d{1,}/g,(s=>{const o={$r:s=>[...Array(s)].map((()=>"r")).join(""),$s:s=>[...Array(s)].map((()=>"s")).join(""),$t:s=>[...Array(s)].map((()=>"t")).join("")},i=s.slice(0,2),a=Number.parseInt(s.slice(2),10);return o[i](a)})).replace(/^(.*?)(t{8,})(.*)$/g,((o,i,a)=>s.substring(i.length,i.length+a.length))):s;if(8===i.length)return new Date(1e3*Number.parseInt(i,16));if(i.length<10)throw new Error("Stamp length invalid");const a=Number.parseInt(i.substring(i.length-1),16);return new Date(1e3*Number.parseInt(i.substring(a,a+8),16))})),__publicField(this,"setCounter",(s=>{this.counter=s})),__publicField(this,"validate",((s,o)=>{const i=o?this._normalizeDictionary(o):this.dict;return s.split("").every((s=>i.includes(s)))}));const o=__spreadValues(__spreadValues({},C),s);this.counter=0,this.debug=!1,this.dict=[],this.version="5.3.2";const{dictionary:i,shuffle:a,length:u,counter:_}=o;this.uuidLength=u,this.setDictionary(i,a),this.setCounter(_),this.debug=o.debug,this.log(this.dict),this.log("Generator instantiated with Dictionary Size ".concat(this.dictLength," and counter set to ").concat(this.counter)),this.log=this.log.bind(this),this.setDictionary=this.setDictionary.bind(this),this.setCounter=this.setCounter.bind(this),this.seq=this.seq.bind(this),this.sequentialUUID=this.sequentialUUID.bind(this),this.rnd=this.rnd.bind(this),this.randomUUID=this.randomUUID.bind(this),this.fmt=this.fmt.bind(this),this.formattedUUID=this.formattedUUID.bind(this),this.availableUUIDs=this.availableUUIDs.bind(this),this.approxMaxBeforeCollision=this.approxMaxBeforeCollision.bind(this),this.collisionProbability=this.collisionProbability.bind(this),this.uniqueness=this.uniqueness.bind(this),this.getVersion=this.getVersion.bind(this),this.stamp=this.stamp.bind(this),this.parseStamp=this.parseStamp.bind(this)}};__publicField(j,"default",j);var L,B=j;return L=w,((a,_,w,x)=>{if(_&&"object"==typeof _||"function"==typeof _)for(let C of i(_))u.call(a,C)||C===w||s(a,C,{get:()=>_[C],enumerable:!(x=o(_,C))||x.enumerable});return a})(s({},"__esModule",{value:!0}),L)})();s.exports=o.default,"undefined"!=typeof window&&(o=o.default)},9325:(s,o,i)=>{var a=i(34840),u="object"==typeof self&&self&&self.Object===Object&&self,_=a||u||Function("return this")();s.exports=_},9404:function(s){s.exports=function(){"use strict";var s=Array.prototype.slice;function createClass(s,o){o&&(s.prototype=Object.create(o.prototype)),s.prototype.constructor=s}function Iterable(s){return isIterable(s)?s:Seq(s)}function KeyedIterable(s){return isKeyed(s)?s:KeyedSeq(s)}function IndexedIterable(s){return isIndexed(s)?s:IndexedSeq(s)}function SetIterable(s){return isIterable(s)&&!isAssociative(s)?s:SetSeq(s)}function isIterable(s){return!(!s||!s[o])}function isKeyed(s){return!(!s||!s[i])}function isIndexed(s){return!(!s||!s[a])}function isAssociative(s){return isKeyed(s)||isIndexed(s)}function isOrdered(s){return!(!s||!s[u])}createClass(KeyedIterable,Iterable),createClass(IndexedIterable,Iterable),createClass(SetIterable,Iterable),Iterable.isIterable=isIterable,Iterable.isKeyed=isKeyed,Iterable.isIndexed=isIndexed,Iterable.isAssociative=isAssociative,Iterable.isOrdered=isOrdered,Iterable.Keyed=KeyedIterable,Iterable.Indexed=IndexedIterable,Iterable.Set=SetIterable;var o="@@__IMMUTABLE_ITERABLE__@@",i="@@__IMMUTABLE_KEYED__@@",a="@@__IMMUTABLE_INDEXED__@@",u="@@__IMMUTABLE_ORDERED__@@",_="delete",w=5,x=1<>>0;if(""+i!==o||4294967295===i)return NaN;o=i}return o<0?ensureSize(s)+o:o}function returnTrue(){return!0}function wholeSlice(s,o,i){return(0===s||void 0!==i&&s<=-i)&&(void 0===o||void 0!==i&&o>=i)}function resolveBegin(s,o){return resolveIndex(s,o,0)}function resolveEnd(s,o){return resolveIndex(s,o,o)}function resolveIndex(s,o,i){return void 0===s?i:s<0?Math.max(0,o+s):void 0===o?s:Math.min(o,s)}var $=0,V=1,U=2,z="function"==typeof Symbol&&Symbol.iterator,Y="@@iterator",Z=z||Y;function Iterator(s){this.next=s}function iteratorValue(s,o,i,a){var u=0===s?o:1===s?i:[o,i];return a?a.value=u:a={value:u,done:!1},a}function iteratorDone(){return{value:void 0,done:!0}}function hasIterator(s){return!!getIteratorFn(s)}function isIterator(s){return s&&"function"==typeof s.next}function getIterator(s){var o=getIteratorFn(s);return o&&o.call(s)}function getIteratorFn(s){var o=s&&(z&&s[z]||s[Y]);if("function"==typeof o)return o}function isArrayLike(s){return s&&"number"==typeof s.length}function Seq(s){return null==s?emptySequence():isIterable(s)?s.toSeq():seqFromValue(s)}function KeyedSeq(s){return null==s?emptySequence().toKeyedSeq():isIterable(s)?isKeyed(s)?s.toSeq():s.fromEntrySeq():keyedSeqFromValue(s)}function IndexedSeq(s){return null==s?emptySequence():isIterable(s)?isKeyed(s)?s.entrySeq():s.toIndexedSeq():indexedSeqFromValue(s)}function SetSeq(s){return(null==s?emptySequence():isIterable(s)?isKeyed(s)?s.entrySeq():s:indexedSeqFromValue(s)).toSetSeq()}Iterator.prototype.toString=function(){return"[Iterator]"},Iterator.KEYS=$,Iterator.VALUES=V,Iterator.ENTRIES=U,Iterator.prototype.inspect=Iterator.prototype.toSource=function(){return this.toString()},Iterator.prototype[Z]=function(){return this},createClass(Seq,Iterable),Seq.of=function(){return Seq(arguments)},Seq.prototype.toSeq=function(){return this},Seq.prototype.toString=function(){return this.__toString("Seq {","}")},Seq.prototype.cacheResult=function(){return!this._cache&&this.__iterateUncached&&(this._cache=this.entrySeq().toArray(),this.size=this._cache.length),this},Seq.prototype.__iterate=function(s,o){return seqIterate(this,s,o,!0)},Seq.prototype.__iterator=function(s,o){return seqIterator(this,s,o,!0)},createClass(KeyedSeq,Seq),KeyedSeq.prototype.toKeyedSeq=function(){return this},createClass(IndexedSeq,Seq),IndexedSeq.of=function(){return IndexedSeq(arguments)},IndexedSeq.prototype.toIndexedSeq=function(){return this},IndexedSeq.prototype.toString=function(){return this.__toString("Seq [","]")},IndexedSeq.prototype.__iterate=function(s,o){return seqIterate(this,s,o,!1)},IndexedSeq.prototype.__iterator=function(s,o){return seqIterator(this,s,o,!1)},createClass(SetSeq,Seq),SetSeq.of=function(){return SetSeq(arguments)},SetSeq.prototype.toSetSeq=function(){return this},Seq.isSeq=isSeq,Seq.Keyed=KeyedSeq,Seq.Set=SetSeq,Seq.Indexed=IndexedSeq;var ee,ie,ae,ce="@@__IMMUTABLE_SEQ__@@";function ArraySeq(s){this._array=s,this.size=s.length}function ObjectSeq(s){var o=Object.keys(s);this._object=s,this._keys=o,this.size=o.length}function IterableSeq(s){this._iterable=s,this.size=s.length||s.size}function IteratorSeq(s){this._iterator=s,this._iteratorCache=[]}function isSeq(s){return!(!s||!s[ce])}function emptySequence(){return ee||(ee=new ArraySeq([]))}function keyedSeqFromValue(s){var o=Array.isArray(s)?new ArraySeq(s).fromEntrySeq():isIterator(s)?new IteratorSeq(s).fromEntrySeq():hasIterator(s)?new IterableSeq(s).fromEntrySeq():"object"==typeof s?new ObjectSeq(s):void 0;if(!o)throw new TypeError("Expected Array or iterable object of [k, v] entries, or keyed object: "+s);return o}function indexedSeqFromValue(s){var o=maybeIndexedSeqFromValue(s);if(!o)throw new TypeError("Expected Array or iterable object of values: "+s);return o}function seqFromValue(s){var o=maybeIndexedSeqFromValue(s)||"object"==typeof s&&new ObjectSeq(s);if(!o)throw new TypeError("Expected Array or iterable object of values, or keyed object: "+s);return o}function maybeIndexedSeqFromValue(s){return isArrayLike(s)?new ArraySeq(s):isIterator(s)?new IteratorSeq(s):hasIterator(s)?new IterableSeq(s):void 0}function seqIterate(s,o,i,a){var u=s._cache;if(u){for(var _=u.length-1,w=0;w<=_;w++){var x=u[i?_-w:w];if(!1===o(x[1],a?x[0]:w,s))return w+1}return w}return s.__iterateUncached(o,i)}function seqIterator(s,o,i,a){var u=s._cache;if(u){var _=u.length-1,w=0;return new Iterator((function(){var s=u[i?_-w:w];return w++>_?iteratorDone():iteratorValue(o,a?s[0]:w-1,s[1])}))}return s.__iteratorUncached(o,i)}function fromJS(s,o){return o?fromJSWith(o,s,"",{"":s}):fromJSDefault(s)}function fromJSWith(s,o,i,a){return Array.isArray(o)?s.call(a,i,IndexedSeq(o).map((function(i,a){return fromJSWith(s,i,a,o)}))):isPlainObj(o)?s.call(a,i,KeyedSeq(o).map((function(i,a){return fromJSWith(s,i,a,o)}))):o}function fromJSDefault(s){return Array.isArray(s)?IndexedSeq(s).map(fromJSDefault).toList():isPlainObj(s)?KeyedSeq(s).map(fromJSDefault).toMap():s}function isPlainObj(s){return s&&(s.constructor===Object||void 0===s.constructor)}function is(s,o){if(s===o||s!=s&&o!=o)return!0;if(!s||!o)return!1;if("function"==typeof s.valueOf&&"function"==typeof o.valueOf){if((s=s.valueOf())===(o=o.valueOf())||s!=s&&o!=o)return!0;if(!s||!o)return!1}return!("function"!=typeof s.equals||"function"!=typeof o.equals||!s.equals(o))}function deepEqual(s,o){if(s===o)return!0;if(!isIterable(o)||void 0!==s.size&&void 0!==o.size&&s.size!==o.size||void 0!==s.__hash&&void 0!==o.__hash&&s.__hash!==o.__hash||isKeyed(s)!==isKeyed(o)||isIndexed(s)!==isIndexed(o)||isOrdered(s)!==isOrdered(o))return!1;if(0===s.size&&0===o.size)return!0;var i=!isAssociative(s);if(isOrdered(s)){var a=s.entries();return o.every((function(s,o){var u=a.next().value;return u&&is(u[1],s)&&(i||is(u[0],o))}))&&a.next().done}var u=!1;if(void 0===s.size)if(void 0===o.size)"function"==typeof s.cacheResult&&s.cacheResult();else{u=!0;var _=s;s=o,o=_}var w=!0,x=o.__iterate((function(o,a){if(i?!s.has(o):u?!is(o,s.get(a,j)):!is(s.get(a,j),o))return w=!1,!1}));return w&&s.size===x}function Repeat(s,o){if(!(this instanceof Repeat))return new Repeat(s,o);if(this._value=s,this.size=void 0===o?1/0:Math.max(0,o),0===this.size){if(ie)return ie;ie=this}}function invariant(s,o){if(!s)throw new Error(o)}function Range(s,o,i){if(!(this instanceof Range))return new Range(s,o,i);if(invariant(0!==i,"Cannot step a Range by 0"),s=s||0,void 0===o&&(o=1/0),i=void 0===i?1:Math.abs(i),oa?iteratorDone():iteratorValue(s,u,i[o?a-u++:u++])}))},createClass(ObjectSeq,KeyedSeq),ObjectSeq.prototype.get=function(s,o){return void 0===o||this.has(s)?this._object[s]:o},ObjectSeq.prototype.has=function(s){return this._object.hasOwnProperty(s)},ObjectSeq.prototype.__iterate=function(s,o){for(var i=this._object,a=this._keys,u=a.length-1,_=0;_<=u;_++){var w=a[o?u-_:_];if(!1===s(i[w],w,this))return _+1}return _},ObjectSeq.prototype.__iterator=function(s,o){var i=this._object,a=this._keys,u=a.length-1,_=0;return new Iterator((function(){var w=a[o?u-_:_];return _++>u?iteratorDone():iteratorValue(s,w,i[w])}))},ObjectSeq.prototype[u]=!0,createClass(IterableSeq,IndexedSeq),IterableSeq.prototype.__iterateUncached=function(s,o){if(o)return this.cacheResult().__iterate(s,o);var i=getIterator(this._iterable),a=0;if(isIterator(i))for(var u;!(u=i.next()).done&&!1!==s(u.value,a++,this););return a},IterableSeq.prototype.__iteratorUncached=function(s,o){if(o)return this.cacheResult().__iterator(s,o);var i=getIterator(this._iterable);if(!isIterator(i))return new Iterator(iteratorDone);var a=0;return new Iterator((function(){var o=i.next();return o.done?o:iteratorValue(s,a++,o.value)}))},createClass(IteratorSeq,IndexedSeq),IteratorSeq.prototype.__iterateUncached=function(s,o){if(o)return this.cacheResult().__iterate(s,o);for(var i,a=this._iterator,u=this._iteratorCache,_=0;_=a.length){var o=i.next();if(o.done)return o;a[u]=o.value}return iteratorValue(s,u,a[u++])}))},createClass(Repeat,IndexedSeq),Repeat.prototype.toString=function(){return 0===this.size?"Repeat []":"Repeat [ "+this._value+" "+this.size+" times ]"},Repeat.prototype.get=function(s,o){return this.has(s)?this._value:o},Repeat.prototype.includes=function(s){return is(this._value,s)},Repeat.prototype.slice=function(s,o){var i=this.size;return wholeSlice(s,o,i)?this:new Repeat(this._value,resolveEnd(o,i)-resolveBegin(s,i))},Repeat.prototype.reverse=function(){return this},Repeat.prototype.indexOf=function(s){return is(this._value,s)?0:-1},Repeat.prototype.lastIndexOf=function(s){return is(this._value,s)?this.size:-1},Repeat.prototype.__iterate=function(s,o){for(var i=0;i=0&&o=0&&ii?iteratorDone():iteratorValue(s,_++,w)}))},Range.prototype.equals=function(s){return s instanceof Range?this._start===s._start&&this._end===s._end&&this._step===s._step:deepEqual(this,s)},createClass(Collection,Iterable),createClass(KeyedCollection,Collection),createClass(IndexedCollection,Collection),createClass(SetCollection,Collection),Collection.Keyed=KeyedCollection,Collection.Indexed=IndexedCollection,Collection.Set=SetCollection;var le="function"==typeof Math.imul&&-2===Math.imul(4294967295,2)?Math.imul:function imul(s,o){var i=65535&(s|=0),a=65535&(o|=0);return i*a+((s>>>16)*a+i*(o>>>16)<<16>>>0)|0};function smi(s){return s>>>1&1073741824|3221225471&s}function hash(s){if(!1===s||null==s)return 0;if("function"==typeof s.valueOf&&(!1===(s=s.valueOf())||null==s))return 0;if(!0===s)return 1;var o=typeof s;if("number"===o){if(s!=s||s===1/0)return 0;var i=0|s;for(i!==s&&(i^=4294967295*s);s>4294967295;)i^=s/=4294967295;return smi(i)}if("string"===o)return s.length>Se?cachedHashString(s):hashString(s);if("function"==typeof s.hashCode)return s.hashCode();if("object"===o)return hashJSObj(s);if("function"==typeof s.toString)return hashString(s.toString());throw new Error("Value type "+o+" cannot be hashed.")}function cachedHashString(s){var o=Pe[s];return void 0===o&&(o=hashString(s),xe===we&&(xe=0,Pe={}),xe++,Pe[s]=o),o}function hashString(s){for(var o=0,i=0;i0)switch(s.nodeType){case 1:return s.uniqueID;case 9:return s.documentElement&&s.documentElement.uniqueID}}var fe,ye="function"==typeof WeakMap;ye&&(fe=new WeakMap);var be=0,_e="__immutablehash__";"function"==typeof Symbol&&(_e=Symbol(_e));var Se=16,we=255,xe=0,Pe={};function assertNotInfinite(s){invariant(s!==1/0,"Cannot perform this action with an infinite size.")}function Map(s){return null==s?emptyMap():isMap(s)&&!isOrdered(s)?s:emptyMap().withMutations((function(o){var i=KeyedIterable(s);assertNotInfinite(i.size),i.forEach((function(s,i){return o.set(i,s)}))}))}function isMap(s){return!(!s||!s[Re])}createClass(Map,KeyedCollection),Map.of=function(){var o=s.call(arguments,0);return emptyMap().withMutations((function(s){for(var i=0;i=o.length)throw new Error("Missing value for key: "+o[i]);s.set(o[i],o[i+1])}}))},Map.prototype.toString=function(){return this.__toString("Map {","}")},Map.prototype.get=function(s,o){return this._root?this._root.get(0,void 0,s,o):o},Map.prototype.set=function(s,o){return updateMap(this,s,o)},Map.prototype.setIn=function(s,o){return this.updateIn(s,j,(function(){return o}))},Map.prototype.remove=function(s){return updateMap(this,s,j)},Map.prototype.deleteIn=function(s){return this.updateIn(s,(function(){return j}))},Map.prototype.update=function(s,o,i){return 1===arguments.length?s(this):this.updateIn([s],o,i)},Map.prototype.updateIn=function(s,o,i){i||(i=o,o=void 0);var a=updateInDeepMap(this,forceIterator(s),o,i);return a===j?void 0:a},Map.prototype.clear=function(){return 0===this.size?this:this.__ownerID?(this.size=0,this._root=null,this.__hash=void 0,this.__altered=!0,this):emptyMap()},Map.prototype.merge=function(){return mergeIntoMapWith(this,void 0,arguments)},Map.prototype.mergeWith=function(o){return mergeIntoMapWith(this,o,s.call(arguments,1))},Map.prototype.mergeIn=function(o){var i=s.call(arguments,1);return this.updateIn(o,emptyMap(),(function(s){return"function"==typeof s.merge?s.merge.apply(s,i):i[i.length-1]}))},Map.prototype.mergeDeep=function(){return mergeIntoMapWith(this,deepMerger,arguments)},Map.prototype.mergeDeepWith=function(o){var i=s.call(arguments,1);return mergeIntoMapWith(this,deepMergerWith(o),i)},Map.prototype.mergeDeepIn=function(o){var i=s.call(arguments,1);return this.updateIn(o,emptyMap(),(function(s){return"function"==typeof s.mergeDeep?s.mergeDeep.apply(s,i):i[i.length-1]}))},Map.prototype.sort=function(s){return OrderedMap(sortFactory(this,s))},Map.prototype.sortBy=function(s,o){return OrderedMap(sortFactory(this,o,s))},Map.prototype.withMutations=function(s){var o=this.asMutable();return s(o),o.wasAltered()?o.__ensureOwner(this.__ownerID):this},Map.prototype.asMutable=function(){return this.__ownerID?this:this.__ensureOwner(new OwnerID)},Map.prototype.asImmutable=function(){return this.__ensureOwner()},Map.prototype.wasAltered=function(){return this.__altered},Map.prototype.__iterator=function(s,o){return new MapIterator(this,s,o)},Map.prototype.__iterate=function(s,o){var i=this,a=0;return this._root&&this._root.iterate((function(o){return a++,s(o[1],o[0],i)}),o),a},Map.prototype.__ensureOwner=function(s){return s===this.__ownerID?this:s?makeMap(this.size,this._root,s,this.__hash):(this.__ownerID=s,this.__altered=!1,this)},Map.isMap=isMap;var Te,Re="@@__IMMUTABLE_MAP__@@",$e=Map.prototype;function ArrayMapNode(s,o){this.ownerID=s,this.entries=o}function BitmapIndexedNode(s,o,i){this.ownerID=s,this.bitmap=o,this.nodes=i}function HashArrayMapNode(s,o,i){this.ownerID=s,this.count=o,this.nodes=i}function HashCollisionNode(s,o,i){this.ownerID=s,this.keyHash=o,this.entries=i}function ValueNode(s,o,i){this.ownerID=s,this.keyHash=o,this.entry=i}function MapIterator(s,o,i){this._type=o,this._reverse=i,this._stack=s._root&&mapIteratorFrame(s._root)}function mapIteratorValue(s,o){return iteratorValue(s,o[0],o[1])}function mapIteratorFrame(s,o){return{node:s,index:0,__prev:o}}function makeMap(s,o,i,a){var u=Object.create($e);return u.size=s,u._root=o,u.__ownerID=i,u.__hash=a,u.__altered=!1,u}function emptyMap(){return Te||(Te=makeMap(0))}function updateMap(s,o,i){var a,u;if(s._root){var _=MakeRef(L),w=MakeRef(B);if(a=updateNode(s._root,s.__ownerID,0,void 0,o,i,_,w),!w.value)return s;u=s.size+(_.value?i===j?-1:1:0)}else{if(i===j)return s;u=1,a=new ArrayMapNode(s.__ownerID,[[o,i]])}return s.__ownerID?(s.size=u,s._root=a,s.__hash=void 0,s.__altered=!0,s):a?makeMap(u,a):emptyMap()}function updateNode(s,o,i,a,u,_,w,x){return s?s.update(o,i,a,u,_,w,x):_===j?s:(SetRef(x),SetRef(w),new ValueNode(o,a,[u,_]))}function isLeafNode(s){return s.constructor===ValueNode||s.constructor===HashCollisionNode}function mergeIntoNode(s,o,i,a,u){if(s.keyHash===a)return new HashCollisionNode(o,a,[s.entry,u]);var _,x=(0===i?s.keyHash:s.keyHash>>>i)&C,j=(0===i?a:a>>>i)&C;return new BitmapIndexedNode(o,1<>>=1)w[C]=1&i?o[_++]:void 0;return w[a]=u,new HashArrayMapNode(s,_+1,w)}function mergeIntoMapWith(s,o,i){for(var a=[],u=0;u>1&1431655765))+(s>>2&858993459))+(s>>4)&252645135,s+=s>>8,127&(s+=s>>16)}function setIn(s,o,i,a){var u=a?s:arrCopy(s);return u[o]=i,u}function spliceIn(s,o,i,a){var u=s.length+1;if(a&&o+1===u)return s[o]=i,s;for(var _=new Array(u),w=0,x=0;x=qe)return createNodes(s,C,a,u);var V=s&&s===this.ownerID,U=V?C:arrCopy(C);return $?x?L===B-1?U.pop():U[L]=U.pop():U[L]=[a,u]:U.push([a,u]),V?(this.entries=U,this):new ArrayMapNode(s,U)}},BitmapIndexedNode.prototype.get=function(s,o,i,a){void 0===o&&(o=hash(i));var u=1<<((0===s?o:o>>>s)&C),_=this.bitmap;return _&u?this.nodes[popCount(_&u-1)].get(s+w,o,i,a):a},BitmapIndexedNode.prototype.update=function(s,o,i,a,u,_,x){void 0===i&&(i=hash(a));var L=(0===o?i:i>>>o)&C,B=1<=ze)return expandNodes(s,z,$,L,Z);if(V&&!Z&&2===z.length&&isLeafNode(z[1^U]))return z[1^U];if(V&&Z&&1===z.length&&isLeafNode(Z))return Z;var ee=s&&s===this.ownerID,ie=V?Z?$:$^B:$|B,ae=V?Z?setIn(z,U,Z,ee):spliceOut(z,U,ee):spliceIn(z,U,Z,ee);return ee?(this.bitmap=ie,this.nodes=ae,this):new BitmapIndexedNode(s,ie,ae)},HashArrayMapNode.prototype.get=function(s,o,i,a){void 0===o&&(o=hash(i));var u=(0===s?o:o>>>s)&C,_=this.nodes[u];return _?_.get(s+w,o,i,a):a},HashArrayMapNode.prototype.update=function(s,o,i,a,u,_,x){void 0===i&&(i=hash(a));var L=(0===o?i:i>>>o)&C,B=u===j,$=this.nodes,V=$[L];if(B&&!V)return this;var U=updateNode(V,s,o+w,i,a,u,_,x);if(U===V)return this;var z=this.count;if(V){if(!U&&--z0&&a=0&&s>>o&C;if(a>=this.array.length)return new VNode([],s);var u,_=0===a;if(o>0){var x=this.array[a];if((u=x&&x.removeBefore(s,o-w,i))===x&&_)return this}if(_&&!u)return this;var j=editableVNode(this,s);if(!_)for(var L=0;L>>o&C;if(u>=this.array.length)return this;if(o>0){var _=this.array[u];if((a=_&&_.removeAfter(s,o-w,i))===_&&u===this.array.length-1)return this}var x=editableVNode(this,s);return x.array.splice(u+1),a&&(x.array[u]=a),x};var Xe,Qe,et={};function iterateList(s,o){var i=s._origin,a=s._capacity,u=getTailOffset(a),_=s._tail;return iterateNodeOrLeaf(s._root,s._level,0);function iterateNodeOrLeaf(s,o,i){return 0===o?iterateLeaf(s,i):iterateNode(s,o,i)}function iterateLeaf(s,w){var C=w===u?_&&_.array:s&&s.array,j=w>i?0:i-w,L=a-w;return L>x&&(L=x),function(){if(j===L)return et;var s=o?--L:j++;return C&&C[s]}}function iterateNode(s,u,_){var C,j=s&&s.array,L=_>i?0:i-_>>u,B=1+(a-_>>u);return B>x&&(B=x),function(){for(;;){if(C){var s=C();if(s!==et)return s;C=null}if(L===B)return et;var i=o?--B:L++;C=iterateNodeOrLeaf(j&&j[i],u-w,_+(i<=s.size||o<0)return s.withMutations((function(s){o<0?setListBounds(s,o).set(0,i):setListBounds(s,0,o+1).set(o,i)}));o+=s._origin;var a=s._tail,u=s._root,_=MakeRef(B);return o>=getTailOffset(s._capacity)?a=updateVNode(a,s.__ownerID,0,o,i,_):u=updateVNode(u,s.__ownerID,s._level,o,i,_),_.value?s.__ownerID?(s._root=u,s._tail=a,s.__hash=void 0,s.__altered=!0,s):makeList(s._origin,s._capacity,s._level,u,a):s}function updateVNode(s,o,i,a,u,_){var x,j=a>>>i&C,L=s&&j0){var B=s&&s.array[j],$=updateVNode(B,o,i-w,a,u,_);return $===B?s:((x=editableVNode(s,o)).array[j]=$,x)}return L&&s.array[j]===u?s:(SetRef(_),x=editableVNode(s,o),void 0===u&&j===x.array.length-1?x.array.pop():x.array[j]=u,x)}function editableVNode(s,o){return o&&s&&o===s.ownerID?s:new VNode(s?s.array.slice():[],o)}function listNodeFor(s,o){if(o>=getTailOffset(s._capacity))return s._tail;if(o<1<0;)i=i.array[o>>>a&C],a-=w;return i}}function setListBounds(s,o,i){void 0!==o&&(o|=0),void 0!==i&&(i|=0);var a=s.__ownerID||new OwnerID,u=s._origin,_=s._capacity,x=u+o,j=void 0===i?_:i<0?_+i:u+i;if(x===u&&j===_)return s;if(x>=j)return s.clear();for(var L=s._level,B=s._root,$=0;x+$<0;)B=new VNode(B&&B.array.length?[void 0,B]:[],a),$+=1<<(L+=w);$&&(x+=$,u+=$,j+=$,_+=$);for(var V=getTailOffset(_),U=getTailOffset(j);U>=1<V?new VNode([],a):z;if(z&&U>V&&x<_&&z.array.length){for(var Z=B=editableVNode(B,a),ee=L;ee>w;ee-=w){var ie=V>>>ee&C;Z=Z.array[ie]=editableVNode(Z.array[ie],a)}Z.array[V>>>w&C]=z}if(j<_&&(Y=Y&&Y.removeAfter(a,0,j)),x>=U)x-=U,j-=U,L=w,B=null,Y=Y&&Y.removeBefore(a,0,x);else if(x>u||U>>L&C;if(ae!==U>>>L&C)break;ae&&($+=(1<u&&(B=B.removeBefore(a,L,x-$)),B&&Uu&&(u=x.size),isIterable(w)||(x=x.map((function(s){return fromJS(s)}))),a.push(x)}return u>s.size&&(s=s.setSize(u)),mergeIntoCollectionWith(s,o,a)}function getTailOffset(s){return s>>w<=x&&w.size>=2*_.size?(a=(u=w.filter((function(s,o){return void 0!==s&&C!==o}))).toKeyedSeq().map((function(s){return s[0]})).flip().toMap(),s.__ownerID&&(a.__ownerID=u.__ownerID=s.__ownerID)):(a=_.remove(o),u=C===w.size-1?w.pop():w.set(C,void 0))}else if(L){if(i===w.get(C)[1])return s;a=_,u=w.set(C,[o,i])}else a=_.set(o,w.size),u=w.set(w.size,[o,i]);return s.__ownerID?(s.size=a.size,s._map=a,s._list=u,s.__hash=void 0,s):makeOrderedMap(a,u)}function ToKeyedSequence(s,o){this._iter=s,this._useKeys=o,this.size=s.size}function ToIndexedSequence(s){this._iter=s,this.size=s.size}function ToSetSequence(s){this._iter=s,this.size=s.size}function FromEntriesSequence(s){this._iter=s,this.size=s.size}function flipFactory(s){var o=makeSequence(s);return o._iter=s,o.size=s.size,o.flip=function(){return s},o.reverse=function(){var o=s.reverse.apply(this);return o.flip=function(){return s.reverse()},o},o.has=function(o){return s.includes(o)},o.includes=function(o){return s.has(o)},o.cacheResult=cacheResultThrough,o.__iterateUncached=function(o,i){var a=this;return s.__iterate((function(s,i){return!1!==o(i,s,a)}),i)},o.__iteratorUncached=function(o,i){if(o===U){var a=s.__iterator(o,i);return new Iterator((function(){var s=a.next();if(!s.done){var o=s.value[0];s.value[0]=s.value[1],s.value[1]=o}return s}))}return s.__iterator(o===V?$:V,i)},o}function mapFactory(s,o,i){var a=makeSequence(s);return a.size=s.size,a.has=function(o){return s.has(o)},a.get=function(a,u){var _=s.get(a,j);return _===j?u:o.call(i,_,a,s)},a.__iterateUncached=function(a,u){var _=this;return s.__iterate((function(s,u,w){return!1!==a(o.call(i,s,u,w),u,_)}),u)},a.__iteratorUncached=function(a,u){var _=s.__iterator(U,u);return new Iterator((function(){var u=_.next();if(u.done)return u;var w=u.value,x=w[0];return iteratorValue(a,x,o.call(i,w[1],x,s),u)}))},a}function reverseFactory(s,o){var i=makeSequence(s);return i._iter=s,i.size=s.size,i.reverse=function(){return s},s.flip&&(i.flip=function(){var o=flipFactory(s);return o.reverse=function(){return s.flip()},o}),i.get=function(i,a){return s.get(o?i:-1-i,a)},i.has=function(i){return s.has(o?i:-1-i)},i.includes=function(o){return s.includes(o)},i.cacheResult=cacheResultThrough,i.__iterate=function(o,i){var a=this;return s.__iterate((function(s,i){return o(s,i,a)}),!i)},i.__iterator=function(o,i){return s.__iterator(o,!i)},i}function filterFactory(s,o,i,a){var u=makeSequence(s);return a&&(u.has=function(a){var u=s.get(a,j);return u!==j&&!!o.call(i,u,a,s)},u.get=function(a,u){var _=s.get(a,j);return _!==j&&o.call(i,_,a,s)?_:u}),u.__iterateUncached=function(u,_){var w=this,x=0;return s.__iterate((function(s,_,C){if(o.call(i,s,_,C))return x++,u(s,a?_:x-1,w)}),_),x},u.__iteratorUncached=function(u,_){var w=s.__iterator(U,_),x=0;return new Iterator((function(){for(;;){var _=w.next();if(_.done)return _;var C=_.value,j=C[0],L=C[1];if(o.call(i,L,j,s))return iteratorValue(u,a?j:x++,L,_)}}))},u}function countByFactory(s,o,i){var a=Map().asMutable();return s.__iterate((function(u,_){a.update(o.call(i,u,_,s),0,(function(s){return s+1}))})),a.asImmutable()}function groupByFactory(s,o,i){var a=isKeyed(s),u=(isOrdered(s)?OrderedMap():Map()).asMutable();s.__iterate((function(_,w){u.update(o.call(i,_,w,s),(function(s){return(s=s||[]).push(a?[w,_]:_),s}))}));var _=iterableClass(s);return u.map((function(o){return reify(s,_(o))}))}function sliceFactory(s,o,i,a){var u=s.size;if(void 0!==o&&(o|=0),void 0!==i&&(i===1/0?i=u:i|=0),wholeSlice(o,i,u))return s;var _=resolveBegin(o,u),w=resolveEnd(i,u);if(_!=_||w!=w)return sliceFactory(s.toSeq().cacheResult(),o,i,a);var x,C=w-_;C==C&&(x=C<0?0:C);var j=makeSequence(s);return j.size=0===x?x:s.size&&x||void 0,!a&&isSeq(s)&&x>=0&&(j.get=function(o,i){return(o=wrapIndex(this,o))>=0&&ox)return iteratorDone();var s=u.next();return a||o===V?s:iteratorValue(o,C-1,o===$?void 0:s.value[1],s)}))},j}function takeWhileFactory(s,o,i){var a=makeSequence(s);return a.__iterateUncached=function(a,u){var _=this;if(u)return this.cacheResult().__iterate(a,u);var w=0;return s.__iterate((function(s,u,x){return o.call(i,s,u,x)&&++w&&a(s,u,_)})),w},a.__iteratorUncached=function(a,u){var _=this;if(u)return this.cacheResult().__iterator(a,u);var w=s.__iterator(U,u),x=!0;return new Iterator((function(){if(!x)return iteratorDone();var s=w.next();if(s.done)return s;var u=s.value,C=u[0],j=u[1];return o.call(i,j,C,_)?a===U?s:iteratorValue(a,C,j,s):(x=!1,iteratorDone())}))},a}function skipWhileFactory(s,o,i,a){var u=makeSequence(s);return u.__iterateUncached=function(u,_){var w=this;if(_)return this.cacheResult().__iterate(u,_);var x=!0,C=0;return s.__iterate((function(s,_,j){if(!x||!(x=o.call(i,s,_,j)))return C++,u(s,a?_:C-1,w)})),C},u.__iteratorUncached=function(u,_){var w=this;if(_)return this.cacheResult().__iterator(u,_);var x=s.__iterator(U,_),C=!0,j=0;return new Iterator((function(){var s,_,L;do{if((s=x.next()).done)return a||u===V?s:iteratorValue(u,j++,u===$?void 0:s.value[1],s);var B=s.value;_=B[0],L=B[1],C&&(C=o.call(i,L,_,w))}while(C);return u===U?s:iteratorValue(u,_,L,s)}))},u}function concatFactory(s,o){var i=isKeyed(s),a=[s].concat(o).map((function(s){return isIterable(s)?i&&(s=KeyedIterable(s)):s=i?keyedSeqFromValue(s):indexedSeqFromValue(Array.isArray(s)?s:[s]),s})).filter((function(s){return 0!==s.size}));if(0===a.length)return s;if(1===a.length){var u=a[0];if(u===s||i&&isKeyed(u)||isIndexed(s)&&isIndexed(u))return u}var _=new ArraySeq(a);return i?_=_.toKeyedSeq():isIndexed(s)||(_=_.toSetSeq()),(_=_.flatten(!0)).size=a.reduce((function(s,o){if(void 0!==s){var i=o.size;if(void 0!==i)return s+i}}),0),_}function flattenFactory(s,o,i){var a=makeSequence(s);return a.__iterateUncached=function(a,u){var _=0,w=!1;function flatDeep(s,x){var C=this;s.__iterate((function(s,u){return(!o||x0}function zipWithFactory(s,o,i){var a=makeSequence(s);return a.size=new ArraySeq(i).map((function(s){return s.size})).min(),a.__iterate=function(s,o){for(var i,a=this.__iterator(V,o),u=0;!(i=a.next()).done&&!1!==s(i.value,u++,this););return u},a.__iteratorUncached=function(s,a){var u=i.map((function(s){return s=Iterable(s),getIterator(a?s.reverse():s)})),_=0,w=!1;return new Iterator((function(){var i;return w||(i=u.map((function(s){return s.next()})),w=i.some((function(s){return s.done}))),w?iteratorDone():iteratorValue(s,_++,o.apply(null,i.map((function(s){return s.value}))))}))},a}function reify(s,o){return isSeq(s)?o:s.constructor(o)}function validateEntry(s){if(s!==Object(s))throw new TypeError("Expected [K, V] tuple: "+s)}function resolveSize(s){return assertNotInfinite(s.size),ensureSize(s)}function iterableClass(s){return isKeyed(s)?KeyedIterable:isIndexed(s)?IndexedIterable:SetIterable}function makeSequence(s){return Object.create((isKeyed(s)?KeyedSeq:isIndexed(s)?IndexedSeq:SetSeq).prototype)}function cacheResultThrough(){return this._iter.cacheResult?(this._iter.cacheResult(),this.size=this._iter.size,this):Seq.prototype.cacheResult.call(this)}function defaultComparator(s,o){return s>o?1:s=0;i--)o={value:arguments[i],next:o};return this.__ownerID?(this.size=s,this._head=o,this.__hash=void 0,this.__altered=!0,this):makeStack(s,o)},Stack.prototype.pushAll=function(s){if(0===(s=IndexedIterable(s)).size)return this;assertNotInfinite(s.size);var o=this.size,i=this._head;return s.reverse().forEach((function(s){o++,i={value:s,next:i}})),this.__ownerID?(this.size=o,this._head=i,this.__hash=void 0,this.__altered=!0,this):makeStack(o,i)},Stack.prototype.pop=function(){return this.slice(1)},Stack.prototype.unshift=function(){return this.push.apply(this,arguments)},Stack.prototype.unshiftAll=function(s){return this.pushAll(s)},Stack.prototype.shift=function(){return this.pop.apply(this,arguments)},Stack.prototype.clear=function(){return 0===this.size?this:this.__ownerID?(this.size=0,this._head=void 0,this.__hash=void 0,this.__altered=!0,this):emptyStack()},Stack.prototype.slice=function(s,o){if(wholeSlice(s,o,this.size))return this;var i=resolveBegin(s,this.size);if(resolveEnd(o,this.size)!==this.size)return IndexedCollection.prototype.slice.call(this,s,o);for(var a=this.size-i,u=this._head;i--;)u=u.next;return this.__ownerID?(this.size=a,this._head=u,this.__hash=void 0,this.__altered=!0,this):makeStack(a,u)},Stack.prototype.__ensureOwner=function(s){return s===this.__ownerID?this:s?makeStack(this.size,this._head,s,this.__hash):(this.__ownerID=s,this.__altered=!1,this)},Stack.prototype.__iterate=function(s,o){if(o)return this.reverse().__iterate(s);for(var i=0,a=this._head;a&&!1!==s(a.value,i++,this);)a=a.next;return i},Stack.prototype.__iterator=function(s,o){if(o)return this.reverse().__iterator(s);var i=0,a=this._head;return new Iterator((function(){if(a){var o=a.value;return a=a.next,iteratorValue(s,i++,o)}return iteratorDone()}))},Stack.isStack=isStack;var at,ct="@@__IMMUTABLE_STACK__@@",lt=Stack.prototype;function makeStack(s,o,i,a){var u=Object.create(lt);return u.size=s,u._head=o,u.__ownerID=i,u.__hash=a,u.__altered=!1,u}function emptyStack(){return at||(at=makeStack(0))}function mixin(s,o){var keyCopier=function(i){s.prototype[i]=o[i]};return Object.keys(o).forEach(keyCopier),Object.getOwnPropertySymbols&&Object.getOwnPropertySymbols(o).forEach(keyCopier),s}lt[ct]=!0,lt.withMutations=$e.withMutations,lt.asMutable=$e.asMutable,lt.asImmutable=$e.asImmutable,lt.wasAltered=$e.wasAltered,Iterable.Iterator=Iterator,mixin(Iterable,{toArray:function(){assertNotInfinite(this.size);var s=new Array(this.size||0);return this.valueSeq().__iterate((function(o,i){s[i]=o})),s},toIndexedSeq:function(){return new ToIndexedSequence(this)},toJS:function(){return this.toSeq().map((function(s){return s&&"function"==typeof s.toJS?s.toJS():s})).__toJS()},toJSON:function(){return this.toSeq().map((function(s){return s&&"function"==typeof s.toJSON?s.toJSON():s})).__toJS()},toKeyedSeq:function(){return new ToKeyedSequence(this,!0)},toMap:function(){return Map(this.toKeyedSeq())},toObject:function(){assertNotInfinite(this.size);var s={};return this.__iterate((function(o,i){s[i]=o})),s},toOrderedMap:function(){return OrderedMap(this.toKeyedSeq())},toOrderedSet:function(){return OrderedSet(isKeyed(this)?this.valueSeq():this)},toSet:function(){return Set(isKeyed(this)?this.valueSeq():this)},toSetSeq:function(){return new ToSetSequence(this)},toSeq:function(){return isIndexed(this)?this.toIndexedSeq():isKeyed(this)?this.toKeyedSeq():this.toSetSeq()},toStack:function(){return Stack(isKeyed(this)?this.valueSeq():this)},toList:function(){return List(isKeyed(this)?this.valueSeq():this)},toString:function(){return"[Iterable]"},__toString:function(s,o){return 0===this.size?s+o:s+" "+this.toSeq().map(this.__toStringMapper).join(", ")+" "+o},concat:function(){return reify(this,concatFactory(this,s.call(arguments,0)))},includes:function(s){return this.some((function(o){return is(o,s)}))},entries:function(){return this.__iterator(U)},every:function(s,o){assertNotInfinite(this.size);var i=!0;return this.__iterate((function(a,u,_){if(!s.call(o,a,u,_))return i=!1,!1})),i},filter:function(s,o){return reify(this,filterFactory(this,s,o,!0))},find:function(s,o,i){var a=this.findEntry(s,o);return a?a[1]:i},forEach:function(s,o){return assertNotInfinite(this.size),this.__iterate(o?s.bind(o):s)},join:function(s){assertNotInfinite(this.size),s=void 0!==s?""+s:",";var o="",i=!0;return this.__iterate((function(a){i?i=!1:o+=s,o+=null!=a?a.toString():""})),o},keys:function(){return this.__iterator($)},map:function(s,o){return reify(this,mapFactory(this,s,o))},reduce:function(s,o,i){var a,u;return assertNotInfinite(this.size),arguments.length<2?u=!0:a=o,this.__iterate((function(o,_,w){u?(u=!1,a=o):a=s.call(i,a,o,_,w)})),a},reduceRight:function(s,o,i){var a=this.toKeyedSeq().reverse();return a.reduce.apply(a,arguments)},reverse:function(){return reify(this,reverseFactory(this,!0))},slice:function(s,o){return reify(this,sliceFactory(this,s,o,!0))},some:function(s,o){return!this.every(not(s),o)},sort:function(s){return reify(this,sortFactory(this,s))},values:function(){return this.__iterator(V)},butLast:function(){return this.slice(0,-1)},isEmpty:function(){return void 0!==this.size?0===this.size:!this.some((function(){return!0}))},count:function(s,o){return ensureSize(s?this.toSeq().filter(s,o):this)},countBy:function(s,o){return countByFactory(this,s,o)},equals:function(s){return deepEqual(this,s)},entrySeq:function(){var s=this;if(s._cache)return new ArraySeq(s._cache);var o=s.toSeq().map(entryMapper).toIndexedSeq();return o.fromEntrySeq=function(){return s.toSeq()},o},filterNot:function(s,o){return this.filter(not(s),o)},findEntry:function(s,o,i){var a=i;return this.__iterate((function(i,u,_){if(s.call(o,i,u,_))return a=[u,i],!1})),a},findKey:function(s,o){var i=this.findEntry(s,o);return i&&i[0]},findLast:function(s,o,i){return this.toKeyedSeq().reverse().find(s,o,i)},findLastEntry:function(s,o,i){return this.toKeyedSeq().reverse().findEntry(s,o,i)},findLastKey:function(s,o){return this.toKeyedSeq().reverse().findKey(s,o)},first:function(){return this.find(returnTrue)},flatMap:function(s,o){return reify(this,flatMapFactory(this,s,o))},flatten:function(s){return reify(this,flattenFactory(this,s,!0))},fromEntrySeq:function(){return new FromEntriesSequence(this)},get:function(s,o){return this.find((function(o,i){return is(i,s)}),void 0,o)},getIn:function(s,o){for(var i,a=this,u=forceIterator(s);!(i=u.next()).done;){var _=i.value;if((a=a&&a.get?a.get(_,j):j)===j)return o}return a},groupBy:function(s,o){return groupByFactory(this,s,o)},has:function(s){return this.get(s,j)!==j},hasIn:function(s){return this.getIn(s,j)!==j},isSubset:function(s){return s="function"==typeof s.includes?s:Iterable(s),this.every((function(o){return s.includes(o)}))},isSuperset:function(s){return(s="function"==typeof s.isSubset?s:Iterable(s)).isSubset(this)},keyOf:function(s){return this.findKey((function(o){return is(o,s)}))},keySeq:function(){return this.toSeq().map(keyMapper).toIndexedSeq()},last:function(){return this.toSeq().reverse().first()},lastKeyOf:function(s){return this.toKeyedSeq().reverse().keyOf(s)},max:function(s){return maxFactory(this,s)},maxBy:function(s,o){return maxFactory(this,o,s)},min:function(s){return maxFactory(this,s?neg(s):defaultNegComparator)},minBy:function(s,o){return maxFactory(this,o?neg(o):defaultNegComparator,s)},rest:function(){return this.slice(1)},skip:function(s){return this.slice(Math.max(0,s))},skipLast:function(s){return reify(this,this.toSeq().reverse().skip(s).reverse())},skipWhile:function(s,o){return reify(this,skipWhileFactory(this,s,o,!0))},skipUntil:function(s,o){return this.skipWhile(not(s),o)},sortBy:function(s,o){return reify(this,sortFactory(this,o,s))},take:function(s){return this.slice(0,Math.max(0,s))},takeLast:function(s){return reify(this,this.toSeq().reverse().take(s).reverse())},takeWhile:function(s,o){return reify(this,takeWhileFactory(this,s,o))},takeUntil:function(s,o){return this.takeWhile(not(s),o)},valueSeq:function(){return this.toIndexedSeq()},hashCode:function(){return this.__hash||(this.__hash=hashIterable(this))}});var ut=Iterable.prototype;ut[o]=!0,ut[Z]=ut.values,ut.__toJS=ut.toArray,ut.__toStringMapper=quoteString,ut.inspect=ut.toSource=function(){return this.toString()},ut.chain=ut.flatMap,ut.contains=ut.includes,mixin(KeyedIterable,{flip:function(){return reify(this,flipFactory(this))},mapEntries:function(s,o){var i=this,a=0;return reify(this,this.toSeq().map((function(u,_){return s.call(o,[_,u],a++,i)})).fromEntrySeq())},mapKeys:function(s,o){var i=this;return reify(this,this.toSeq().flip().map((function(a,u){return s.call(o,a,u,i)})).flip())}});var pt=KeyedIterable.prototype;function keyMapper(s,o){return o}function entryMapper(s,o){return[o,s]}function not(s){return function(){return!s.apply(this,arguments)}}function neg(s){return function(){return-s.apply(this,arguments)}}function quoteString(s){return"string"==typeof s?JSON.stringify(s):String(s)}function defaultZipper(){return arrCopy(arguments)}function defaultNegComparator(s,o){return so?-1:0}function hashIterable(s){if(s.size===1/0)return 0;var o=isOrdered(s),i=isKeyed(s),a=o?1:0;return murmurHashOfSize(s.__iterate(i?o?function(s,o){a=31*a+hashMerge(hash(s),hash(o))|0}:function(s,o){a=a+hashMerge(hash(s),hash(o))|0}:o?function(s){a=31*a+hash(s)|0}:function(s){a=a+hash(s)|0}),a)}function murmurHashOfSize(s,o){return o=le(o,3432918353),o=le(o<<15|o>>>-15,461845907),o=le(o<<13|o>>>-13,5),o=le((o=o+3864292196^s)^o>>>16,2246822507),o=smi((o=le(o^o>>>13,3266489909))^o>>>16)}function hashMerge(s,o){return s^o+2654435769+(s<<6)+(s>>2)}return pt[i]=!0,pt[Z]=ut.entries,pt.__toJS=ut.toObject,pt.__toStringMapper=function(s,o){return JSON.stringify(o)+": "+quoteString(s)},mixin(IndexedIterable,{toKeyedSeq:function(){return new ToKeyedSequence(this,!1)},filter:function(s,o){return reify(this,filterFactory(this,s,o,!1))},findIndex:function(s,o){var i=this.findEntry(s,o);return i?i[0]:-1},indexOf:function(s){var o=this.keyOf(s);return void 0===o?-1:o},lastIndexOf:function(s){var o=this.lastKeyOf(s);return void 0===o?-1:o},reverse:function(){return reify(this,reverseFactory(this,!1))},slice:function(s,o){return reify(this,sliceFactory(this,s,o,!1))},splice:function(s,o){var i=arguments.length;if(o=Math.max(0|o,0),0===i||2===i&&!o)return this;s=resolveBegin(s,s<0?this.count():this.size);var a=this.slice(0,s);return reify(this,1===i?a:a.concat(arrCopy(arguments,2),this.slice(s+o)))},findLastIndex:function(s,o){var i=this.findLastEntry(s,o);return i?i[0]:-1},first:function(){return this.get(0)},flatten:function(s){return reify(this,flattenFactory(this,s,!1))},get:function(s,o){return(s=wrapIndex(this,s))<0||this.size===1/0||void 0!==this.size&&s>this.size?o:this.find((function(o,i){return i===s}),void 0,o)},has:function(s){return(s=wrapIndex(this,s))>=0&&(void 0!==this.size?this.size===1/0||s{"use strict";i(71340);var a=i(92046);s.exports=a.Object.assign},9999:(s,o,i)=>{var a=i(37217),u=i(83729),_=i(16547),w=i(74733),x=i(43838),C=i(93290),j=i(23007),L=i(92271),B=i(48948),$=i(50002),V=i(83349),U=i(5861),z=i(76189),Y=i(77199),Z=i(35529),ee=i(56449),ie=i(3656),ae=i(87730),ce=i(23805),le=i(38440),pe=i(95950),de=i(37241),fe="[object Arguments]",ye="[object Function]",be="[object Object]",_e={};_e[fe]=_e["[object Array]"]=_e["[object ArrayBuffer]"]=_e["[object DataView]"]=_e["[object Boolean]"]=_e["[object Date]"]=_e["[object Float32Array]"]=_e["[object Float64Array]"]=_e["[object Int8Array]"]=_e["[object Int16Array]"]=_e["[object Int32Array]"]=_e["[object Map]"]=_e["[object Number]"]=_e[be]=_e["[object RegExp]"]=_e["[object Set]"]=_e["[object String]"]=_e["[object Symbol]"]=_e["[object Uint8Array]"]=_e["[object Uint8ClampedArray]"]=_e["[object Uint16Array]"]=_e["[object Uint32Array]"]=!0,_e["[object Error]"]=_e[ye]=_e["[object WeakMap]"]=!1,s.exports=function baseClone(s,o,i,Se,we,xe){var Pe,Te=1&o,Re=2&o,$e=4&o;if(i&&(Pe=we?i(s,Se,we,xe):i(s)),void 0!==Pe)return Pe;if(!ce(s))return s;var qe=ee(s);if(qe){if(Pe=z(s),!Te)return j(s,Pe)}else{var ze=U(s),We=ze==ye||"[object GeneratorFunction]"==ze;if(ie(s))return C(s,Te);if(ze==be||ze==fe||We&&!we){if(Pe=Re||We?{}:Z(s),!Te)return Re?B(s,x(Pe,s)):L(s,w(Pe,s))}else{if(!_e[ze])return we?s:{};Pe=Y(s,ze,Te)}}xe||(xe=new a);var He=xe.get(s);if(He)return He;xe.set(s,Pe),le(s)?s.forEach((function(a){Pe.add(baseClone(a,o,i,a,s,xe))})):ae(s)&&s.forEach((function(a,u){Pe.set(u,baseClone(a,o,i,u,s,xe))}));var Ye=qe?void 0:($e?Re?V:$:Re?de:pe)(s);return u(Ye||s,(function(a,u){Ye&&(a=s[u=a]),_(Pe,u,baseClone(a,o,i,u,s,xe))})),Pe}},10023:(s,o,i)=>{const a=i(6205),INTS=()=>[{type:a.RANGE,from:48,to:57}],WORDS=()=>[{type:a.CHAR,value:95},{type:a.RANGE,from:97,to:122},{type:a.RANGE,from:65,to:90}].concat(INTS()),WHITESPACE=()=>[{type:a.CHAR,value:9},{type:a.CHAR,value:10},{type:a.CHAR,value:11},{type:a.CHAR,value:12},{type:a.CHAR,value:13},{type:a.CHAR,value:32},{type:a.CHAR,value:160},{type:a.CHAR,value:5760},{type:a.RANGE,from:8192,to:8202},{type:a.CHAR,value:8232},{type:a.CHAR,value:8233},{type:a.CHAR,value:8239},{type:a.CHAR,value:8287},{type:a.CHAR,value:12288},{type:a.CHAR,value:65279}];o.words=()=>({type:a.SET,set:WORDS(),not:!1}),o.notWords=()=>({type:a.SET,set:WORDS(),not:!0}),o.ints=()=>({type:a.SET,set:INTS(),not:!1}),o.notInts=()=>({type:a.SET,set:INTS(),not:!0}),o.whitespace=()=>({type:a.SET,set:WHITESPACE(),not:!1}),o.notWhitespace=()=>({type:a.SET,set:WHITESPACE(),not:!0}),o.anyChar=()=>({type:a.SET,set:[{type:a.CHAR,value:10},{type:a.CHAR,value:13},{type:a.CHAR,value:8232},{type:a.CHAR,value:8233}],not:!0})},10043:(s,o,i)=>{"use strict";var a=i(54018),u=String,_=TypeError;s.exports=function(s){if(a(s))return s;throw new _("Can't set "+u(s)+" as a prototype")}},10124:(s,o,i)=>{var a=i(9325);s.exports=function(){return a.Date.now()}},10300:(s,o,i)=>{"use strict";var a=i(13930),u=i(82159),_=i(36624),w=i(4640),x=i(73448),C=TypeError;s.exports=function(s,o){var i=arguments.length<2?x(s):o;if(u(i))return _(a(i,s));throw new C(w(s)+" is not iterable")}},10316:(s,o,i)=>{const a=i(2404),u=i(55973),_=i(92340);class Element{constructor(s,o,i){o&&(this.meta=o),i&&(this.attributes=i),this.content=s}freeze(){Object.isFrozen(this)||(this._meta&&(this.meta.parent=this,this.meta.freeze()),this._attributes&&(this.attributes.parent=this,this.attributes.freeze()),this.children.forEach((s=>{s.parent=this,s.freeze()}),this),this.content&&Array.isArray(this.content)&&Object.freeze(this.content),Object.freeze(this))}primitive(){}clone(){const s=new this.constructor;return s.element=this.element,this.meta.length&&(s._meta=this.meta.clone()),this.attributes.length&&(s._attributes=this.attributes.clone()),this.content?this.content.clone?s.content=this.content.clone():Array.isArray(this.content)?s.content=this.content.map((s=>s.clone())):s.content=this.content:s.content=this.content,s}toValue(){return this.content instanceof Element?this.content.toValue():this.content instanceof u?{key:this.content.key.toValue(),value:this.content.value?this.content.value.toValue():void 0}:this.content&&this.content.map?this.content.map((s=>s.toValue()),this):this.content}toRef(s){if(""===this.id.toValue())throw Error("Cannot create reference to an element that does not contain an ID");const o=new this.RefElement(this.id.toValue());return s&&(o.path=s),o}findRecursive(...s){if(arguments.length>1&&!this.isFrozen)throw new Error("Cannot find recursive with multiple element names without first freezing the element. Call `element.freeze()`");const o=s.pop();let i=new _;const append=(s,o)=>(s.push(o),s),checkElement=(s,i)=>{i.element===o&&s.push(i);const a=i.findRecursive(o);return a&&a.reduce(append,s),i.content instanceof u&&(i.content.key&&checkElement(s,i.content.key),i.content.value&&checkElement(s,i.content.value)),s};return this.content&&(this.content.element&&checkElement(i,this.content),Array.isArray(this.content)&&this.content.reduce(checkElement,i)),s.isEmpty||(i=i.filter((o=>{let i=o.parents.map((s=>s.element));for(const o in s){const a=s[o],u=i.indexOf(a);if(-1===u)return!1;i=i.splice(0,u)}return!0}))),i}set(s){return this.content=s,this}equals(s){return a(this.toValue(),s)}getMetaProperty(s,o){if(!this.meta.hasKey(s)){if(this.isFrozen){const s=this.refract(o);return s.freeze(),s}this.meta.set(s,o)}return this.meta.get(s)}setMetaProperty(s,o){this.meta.set(s,o)}get element(){return this._storedElement||"element"}set element(s){this._storedElement=s}get content(){return this._content}set content(s){if(s instanceof Element)this._content=s;else if(s instanceof _)this.content=s.elements;else if("string"==typeof s||"number"==typeof s||"boolean"==typeof s||"null"===s||null==s)this._content=s;else if(s instanceof u)this._content=s;else if(Array.isArray(s))this._content=s.map(this.refract);else{if("object"!=typeof s)throw new Error("Cannot set content to given value");this._content=Object.keys(s).map((o=>new this.MemberElement(o,s[o])))}}get meta(){if(!this._meta){if(this.isFrozen){const s=new this.ObjectElement;return s.freeze(),s}this._meta=new this.ObjectElement}return this._meta}set meta(s){s instanceof this.ObjectElement?this._meta=s:this.meta.set(s||{})}get attributes(){if(!this._attributes){if(this.isFrozen){const s=new this.ObjectElement;return s.freeze(),s}this._attributes=new this.ObjectElement}return this._attributes}set attributes(s){s instanceof this.ObjectElement?this._attributes=s:this.attributes.set(s||{})}get id(){return this.getMetaProperty("id","")}set id(s){this.setMetaProperty("id",s)}get classes(){return this.getMetaProperty("classes",[])}set classes(s){this.setMetaProperty("classes",s)}get title(){return this.getMetaProperty("title","")}set title(s){this.setMetaProperty("title",s)}get description(){return this.getMetaProperty("description","")}set description(s){this.setMetaProperty("description",s)}get links(){return this.getMetaProperty("links",[])}set links(s){this.setMetaProperty("links",s)}get isFrozen(){return Object.isFrozen(this)}get parents(){let{parent:s}=this;const o=new _;for(;s;)o.push(s),s=s.parent;return o}get children(){if(Array.isArray(this.content))return new _(this.content);if(this.content instanceof u){const s=new _([this.content.key]);return this.content.value&&s.push(this.content.value),s}return this.content instanceof Element?new _([this.content]):new _}get recursiveChildren(){const s=new _;return this.children.forEach((o=>{s.push(o),o.recursiveChildren.forEach((o=>{s.push(o)}))})),s}}s.exports=Element},10392:s=>{s.exports=function getValue(s,o){return null==s?void 0:s[o]}},10776:(s,o,i)=>{var a=i(30756),u=i(95950);s.exports=function getMatchData(s){for(var o=u(s),i=o.length;i--;){var _=o[i],w=s[_];o[i]=[_,w,a(w)]}return o}},10866:(s,o,i)=>{const a=i(6048),u=i(92340);class ObjectSlice extends u{map(s,o){return this.elements.map((i=>s.bind(o)(i.value,i.key,i)))}filter(s,o){return new ObjectSlice(this.elements.filter((i=>s.bind(o)(i.value,i.key,i))))}reject(s,o){return this.filter(a(s.bind(o)))}forEach(s,o){return this.elements.forEach(((i,a)=>{s.bind(o)(i.value,i.key,i,a)}))}keys(){return this.map(((s,o)=>o.toValue()))}values(){return this.map((s=>s.toValue()))}}s.exports=ObjectSlice},11042:(s,o,i)=>{"use strict";var a=i(85582),u=i(1907),_=i(24443),w=i(87170),x=i(36624),C=u([].concat);s.exports=a("Reflect","ownKeys")||function ownKeys(s){var o=_.f(x(s)),i=w.f;return i?C(o,i(s)):o}},11091:(s,o,i)=>{"use strict";var a=i(45951),u=i(76024),_=i(92361),w=i(62250),x=i(13846).f,C=i(7463),j=i(92046),L=i(28311),B=i(61626),$=i(49724);i(36128);var wrapConstructor=function(s){var Wrapper=function(o,i,a){if(this instanceof Wrapper){switch(arguments.length){case 0:return new s;case 1:return new s(o);case 2:return new s(o,i)}return new s(o,i,a)}return u(s,this,arguments)};return Wrapper.prototype=s.prototype,Wrapper};s.exports=function(s,o){var i,u,V,U,z,Y,Z,ee,ie,ae=s.target,ce=s.global,le=s.stat,pe=s.proto,de=ce?a:le?a[ae]:a[ae]&&a[ae].prototype,fe=ce?j:j[ae]||B(j,ae,{})[ae],ye=fe.prototype;for(U in o)u=!(i=C(ce?U:ae+(le?".":"#")+U,s.forced))&&de&&$(de,U),Y=fe[U],u&&(Z=s.dontCallGetSet?(ie=x(de,U))&&ie.value:de[U]),z=u&&Z?Z:o[U],(i||pe||typeof Y!=typeof z)&&(ee=s.bind&&u?L(z,a):s.wrap&&u?wrapConstructor(z):pe&&w(z)?_(z):z,(s.sham||z&&z.sham||Y&&Y.sham)&&B(ee,"sham",!0),B(fe,U,ee),pe&&($(j,V=ae+"Prototype")||B(j,V,{}),B(j[V],U,z),s.real&&ye&&(i||!ye[U])&&B(ye,U,z)))}},11287:s=>{s.exports=function getHolder(s){return s.placeholder}},11331:(s,o,i)=>{var a=i(72552),u=i(28879),_=i(40346),w=Function.prototype,x=Object.prototype,C=w.toString,j=x.hasOwnProperty,L=C.call(Object);s.exports=function isPlainObject(s){if(!_(s)||"[object Object]"!=a(s))return!1;var o=u(s);if(null===o)return!0;var i=j.call(o,"constructor")&&o.constructor;return"function"==typeof i&&i instanceof i&&C.call(i)==L}},11470:(s,o,i)=>{"use strict";var a=i(1907),u=i(65482),_=i(90160),w=i(74239),x=a("".charAt),C=a("".charCodeAt),j=a("".slice),createMethod=function(s){return function(o,i){var a,L,B=_(w(o)),$=u(i),V=B.length;return $<0||$>=V?s?"":void 0:(a=C(B,$))<55296||a>56319||$+1===V||(L=C(B,$+1))<56320||L>57343?s?x(B,$):a:s?j(B,$,$+2):L-56320+(a-55296<<10)+65536}};s.exports={codeAt:createMethod(!1),charAt:createMethod(!0)}},11842:(s,o,i)=>{var a=i(82819),u=i(9325);s.exports=function createBind(s,o,i){var _=1&o,w=a(s);return function wrapper(){return(this&&this!==u&&this instanceof wrapper?w:s).apply(_?i:this,arguments)}}},12242:(s,o,i)=>{const a=i(10316);s.exports=class BooleanElement extends a{constructor(s,o,i){super(s,o,i),this.element="boolean"}primitive(){return"boolean"}}},12507:(s,o,i)=>{var a=i(28754),u=i(49698),_=i(63912),w=i(13222);s.exports=function createCaseFirst(s){return function(o){o=w(o);var i=u(o)?_(o):void 0,x=i?i[0]:o.charAt(0),C=i?a(i,1).join(""):o.slice(1);return x[s]()+C}}},12560:(s,o,i)=>{"use strict";i(99363);var a=i(19287),u=i(45951),_=i(14840),w=i(93742);for(var x in a)_(u[x],x),w[x]=w.Array},12651:(s,o,i)=>{var a=i(74218);s.exports=function getMapData(s,o){var i=s.__data__;return a(o)?i["string"==typeof o?"string":"hash"]:i.map}},12749:(s,o,i)=>{var a=i(81042),u=Object.prototype.hasOwnProperty;s.exports=function hashHas(s){var o=this.__data__;return a?void 0!==o[s]:u.call(o,s)}},13222:(s,o,i)=>{var a=i(77556);s.exports=function toString(s){return null==s?"":a(s)}},13846:(s,o,i)=>{"use strict";var a=i(39447),u=i(13930),_=i(22574),w=i(75817),x=i(4993),C=i(70470),j=i(49724),L=i(73648),B=Object.getOwnPropertyDescriptor;o.f=a?B:function getOwnPropertyDescriptor(s,o){if(s=x(s),o=C(o),L)try{return B(s,o)}catch(s){}if(j(s,o))return w(!u(_.f,s,o),s[o])}},13930:(s,o,i)=>{"use strict";var a=i(41505),u=Function.prototype.call;s.exports=a?u.bind(u):function(){return u.apply(u,arguments)}},14248:s=>{s.exports=function arraySome(s,o){for(var i=-1,a=null==s?0:s.length;++i{s.exports=function arrayPush(s,o){for(var i=-1,a=o.length,u=s.length;++i{const a=i(10316);s.exports=class RefElement extends a{constructor(s,o,i){super(s||[],o,i),this.element="ref",this.path||(this.path="element")}get path(){return this.attributes.get("path")}set path(s){this.attributes.set("path",s)}}},14744:s=>{"use strict";var o=function isMergeableObject(s){return function isNonNullObject(s){return!!s&&"object"==typeof s}(s)&&!function isSpecial(s){var o=Object.prototype.toString.call(s);return"[object RegExp]"===o||"[object Date]"===o||function isReactElement(s){return s.$$typeof===i}(s)}(s)};var i="function"==typeof Symbol&&Symbol.for?Symbol.for("react.element"):60103;function cloneUnlessOtherwiseSpecified(s,o){return!1!==o.clone&&o.isMergeableObject(s)?deepmerge(function emptyTarget(s){return Array.isArray(s)?[]:{}}(s),s,o):s}function defaultArrayMerge(s,o,i){return s.concat(o).map((function(s){return cloneUnlessOtherwiseSpecified(s,i)}))}function getKeys(s){return Object.keys(s).concat(function getEnumerableOwnPropertySymbols(s){return Object.getOwnPropertySymbols?Object.getOwnPropertySymbols(s).filter((function(o){return Object.propertyIsEnumerable.call(s,o)})):[]}(s))}function propertyIsOnObject(s,o){try{return o in s}catch(s){return!1}}function mergeObject(s,o,i){var a={};return i.isMergeableObject(s)&&getKeys(s).forEach((function(o){a[o]=cloneUnlessOtherwiseSpecified(s[o],i)})),getKeys(o).forEach((function(u){(function propertyIsUnsafe(s,o){return propertyIsOnObject(s,o)&&!(Object.hasOwnProperty.call(s,o)&&Object.propertyIsEnumerable.call(s,o))})(s,u)||(propertyIsOnObject(s,u)&&i.isMergeableObject(o[u])?a[u]=function getMergeFunction(s,o){if(!o.customMerge)return deepmerge;var i=o.customMerge(s);return"function"==typeof i?i:deepmerge}(u,i)(s[u],o[u],i):a[u]=cloneUnlessOtherwiseSpecified(o[u],i))})),a}function deepmerge(s,i,a){(a=a||{}).arrayMerge=a.arrayMerge||defaultArrayMerge,a.isMergeableObject=a.isMergeableObject||o,a.cloneUnlessOtherwiseSpecified=cloneUnlessOtherwiseSpecified;var u=Array.isArray(i);return u===Array.isArray(s)?u?a.arrayMerge(s,i,a):mergeObject(s,i,a):cloneUnlessOtherwiseSpecified(i,a)}deepmerge.all=function deepmergeAll(s,o){if(!Array.isArray(s))throw new Error("first argument should be an array");return s.reduce((function(s,i){return deepmerge(s,i,o)}),{})};var a=deepmerge;s.exports=a},14792:(s,o,i)=>{var a=i(13222),u=i(55808);s.exports=function capitalize(s){return u(a(s).toLowerCase())}},14840:(s,o,i)=>{"use strict";var a=i(52623),u=i(74284).f,_=i(61626),w=i(49724),x=i(54878),C=i(76264)("toStringTag");s.exports=function(s,o,i,j){var L=i?s:s&&s.prototype;L&&(w(L,C)||u(L,C,{configurable:!0,value:o}),j&&!a&&_(L,"toString",x))}},14974:s=>{s.exports=function safeGet(s,o){if(("constructor"!==o||"function"!=typeof s[o])&&"__proto__"!=o)return s[o]}},15287:(s,o)=>{"use strict";var i=Symbol.for("react.element"),a=Symbol.for("react.portal"),u=Symbol.for("react.fragment"),_=Symbol.for("react.strict_mode"),w=Symbol.for("react.profiler"),x=Symbol.for("react.provider"),C=Symbol.for("react.context"),j=Symbol.for("react.forward_ref"),L=Symbol.for("react.suspense"),B=Symbol.for("react.memo"),$=Symbol.for("react.lazy"),V=Symbol.iterator;var U={isMounted:function(){return!1},enqueueForceUpdate:function(){},enqueueReplaceState:function(){},enqueueSetState:function(){}},z=Object.assign,Y={};function E(s,o,i){this.props=s,this.context=o,this.refs=Y,this.updater=i||U}function F(){}function G(s,o,i){this.props=s,this.context=o,this.refs=Y,this.updater=i||U}E.prototype.isReactComponent={},E.prototype.setState=function(s,o){if("object"!=typeof s&&"function"!=typeof s&&null!=s)throw Error("setState(...): takes an object of state variables to update or a function which returns an object of state variables.");this.updater.enqueueSetState(this,s,o,"setState")},E.prototype.forceUpdate=function(s){this.updater.enqueueForceUpdate(this,s,"forceUpdate")},F.prototype=E.prototype;var Z=G.prototype=new F;Z.constructor=G,z(Z,E.prototype),Z.isPureReactComponent=!0;var ee=Array.isArray,ie=Object.prototype.hasOwnProperty,ae={current:null},ce={key:!0,ref:!0,__self:!0,__source:!0};function M(s,o,a){var u,_={},w=null,x=null;if(null!=o)for(u in void 0!==o.ref&&(x=o.ref),void 0!==o.key&&(w=""+o.key),o)ie.call(o,u)&&!ce.hasOwnProperty(u)&&(_[u]=o[u]);var C=arguments.length-2;if(1===C)_.children=a;else if(1{var a=i(96131);s.exports=function arrayIncludes(s,o){return!!(null==s?0:s.length)&&a(s,o,0)>-1}},15340:()=>{},15389:(s,o,i)=>{var a=i(93663),u=i(87978),_=i(83488),w=i(56449),x=i(50583);s.exports=function baseIteratee(s){return"function"==typeof s?s:null==s?_:"object"==typeof s?w(s)?u(s[0],s[1]):a(s):x(s)}},15972:(s,o,i)=>{"use strict";var a=i(49724),u=i(62250),_=i(39298),w=i(92522),x=i(57382),C=w("IE_PROTO"),j=Object,L=j.prototype;s.exports=x?j.getPrototypeOf:function(s){var o=_(s);if(a(o,C))return o[C];var i=o.constructor;return u(i)&&o instanceof i?i.prototype:o instanceof j?L:null}},16038:(s,o,i)=>{var a=i(5861),u=i(40346);s.exports=function baseIsSet(s){return u(s)&&"[object Set]"==a(s)}},16426:s=>{s.exports=function(){var s=document.getSelection();if(!s.rangeCount)return function(){};for(var o=document.activeElement,i=[],a=0;a{var a=i(43360),u=i(75288),_=Object.prototype.hasOwnProperty;s.exports=function assignValue(s,o,i){var w=s[o];_.call(s,o)&&u(w,i)&&(void 0!==i||o in s)||a(s,o,i)}},16708:(s,o,i)=>{"use strict";var a,u=i(65606);function CorkedRequest(s){var o=this;this.next=null,this.entry=null,this.finish=function(){!function onCorkedFinish(s,o,i){var a=s.entry;s.entry=null;for(;a;){var u=a.callback;o.pendingcb--,u(i),a=a.next}o.corkedRequestsFree.next=s}(o,s)}}s.exports=Writable,Writable.WritableState=WritableState;var _={deprecate:i(94643)},w=i(40345),x=i(48287).Buffer,C=(void 0!==i.g?i.g:"undefined"!=typeof window?window:"undefined"!=typeof self?self:{}).Uint8Array||function(){};var j,L=i(75896),B=i(65291).getHighWaterMark,$=i(86048).F,V=$.ERR_INVALID_ARG_TYPE,U=$.ERR_METHOD_NOT_IMPLEMENTED,z=$.ERR_MULTIPLE_CALLBACK,Y=$.ERR_STREAM_CANNOT_PIPE,Z=$.ERR_STREAM_DESTROYED,ee=$.ERR_STREAM_NULL_VALUES,ie=$.ERR_STREAM_WRITE_AFTER_END,ae=$.ERR_UNKNOWN_ENCODING,ce=L.errorOrDestroy;function nop(){}function WritableState(s,o,_){a=a||i(25382),s=s||{},"boolean"!=typeof _&&(_=o instanceof a),this.objectMode=!!s.objectMode,_&&(this.objectMode=this.objectMode||!!s.writableObjectMode),this.highWaterMark=B(this,s,"writableHighWaterMark",_),this.finalCalled=!1,this.needDrain=!1,this.ending=!1,this.ended=!1,this.finished=!1,this.destroyed=!1;var w=!1===s.decodeStrings;this.decodeStrings=!w,this.defaultEncoding=s.defaultEncoding||"utf8",this.length=0,this.writing=!1,this.corked=0,this.sync=!0,this.bufferProcessing=!1,this.onwrite=function(s){!function onwrite(s,o){var i=s._writableState,a=i.sync,_=i.writecb;if("function"!=typeof _)throw new z;if(function onwriteStateUpdate(s){s.writing=!1,s.writecb=null,s.length-=s.writelen,s.writelen=0}(i),o)!function onwriteError(s,o,i,a,_){--o.pendingcb,i?(u.nextTick(_,a),u.nextTick(finishMaybe,s,o),s._writableState.errorEmitted=!0,ce(s,a)):(_(a),s._writableState.errorEmitted=!0,ce(s,a),finishMaybe(s,o))}(s,i,a,o,_);else{var w=needFinish(i)||s.destroyed;w||i.corked||i.bufferProcessing||!i.bufferedRequest||clearBuffer(s,i),a?u.nextTick(afterWrite,s,i,w,_):afterWrite(s,i,w,_)}}(o,s)},this.writecb=null,this.writelen=0,this.bufferedRequest=null,this.lastBufferedRequest=null,this.pendingcb=0,this.prefinished=!1,this.errorEmitted=!1,this.emitClose=!1!==s.emitClose,this.autoDestroy=!!s.autoDestroy,this.bufferedRequestCount=0,this.corkedRequestsFree=new CorkedRequest(this)}function Writable(s){var o=this instanceof(a=a||i(25382));if(!o&&!j.call(Writable,this))return new Writable(s);this._writableState=new WritableState(s,this,o),this.writable=!0,s&&("function"==typeof s.write&&(this._write=s.write),"function"==typeof s.writev&&(this._writev=s.writev),"function"==typeof s.destroy&&(this._destroy=s.destroy),"function"==typeof s.final&&(this._final=s.final)),w.call(this)}function doWrite(s,o,i,a,u,_,w){o.writelen=a,o.writecb=w,o.writing=!0,o.sync=!0,o.destroyed?o.onwrite(new Z("write")):i?s._writev(u,o.onwrite):s._write(u,_,o.onwrite),o.sync=!1}function afterWrite(s,o,i,a){i||function onwriteDrain(s,o){0===o.length&&o.needDrain&&(o.needDrain=!1,s.emit("drain"))}(s,o),o.pendingcb--,a(),finishMaybe(s,o)}function clearBuffer(s,o){o.bufferProcessing=!0;var i=o.bufferedRequest;if(s._writev&&i&&i.next){var a=o.bufferedRequestCount,u=new Array(a),_=o.corkedRequestsFree;_.entry=i;for(var w=0,x=!0;i;)u[w]=i,i.isBuf||(x=!1),i=i.next,w+=1;u.allBuffers=x,doWrite(s,o,!0,o.length,u,"",_.finish),o.pendingcb++,o.lastBufferedRequest=null,_.next?(o.corkedRequestsFree=_.next,_.next=null):o.corkedRequestsFree=new CorkedRequest(o),o.bufferedRequestCount=0}else{for(;i;){var C=i.chunk,j=i.encoding,L=i.callback;if(doWrite(s,o,!1,o.objectMode?1:C.length,C,j,L),i=i.next,o.bufferedRequestCount--,o.writing)break}null===i&&(o.lastBufferedRequest=null)}o.bufferedRequest=i,o.bufferProcessing=!1}function needFinish(s){return s.ending&&0===s.length&&null===s.bufferedRequest&&!s.finished&&!s.writing}function callFinal(s,o){s._final((function(i){o.pendingcb--,i&&ce(s,i),o.prefinished=!0,s.emit("prefinish"),finishMaybe(s,o)}))}function finishMaybe(s,o){var i=needFinish(o);if(i&&(function prefinish(s,o){o.prefinished||o.finalCalled||("function"!=typeof s._final||o.destroyed?(o.prefinished=!0,s.emit("prefinish")):(o.pendingcb++,o.finalCalled=!0,u.nextTick(callFinal,s,o)))}(s,o),0===o.pendingcb&&(o.finished=!0,s.emit("finish"),o.autoDestroy))){var a=s._readableState;(!a||a.autoDestroy&&a.endEmitted)&&s.destroy()}return i}i(56698)(Writable,w),WritableState.prototype.getBuffer=function getBuffer(){for(var s=this.bufferedRequest,o=[];s;)o.push(s),s=s.next;return o},function(){try{Object.defineProperty(WritableState.prototype,"buffer",{get:_.deprecate((function writableStateBufferGetter(){return this.getBuffer()}),"_writableState.buffer is deprecated. Use _writableState.getBuffer instead.","DEP0003")})}catch(s){}}(),"function"==typeof Symbol&&Symbol.hasInstance&&"function"==typeof Function.prototype[Symbol.hasInstance]?(j=Function.prototype[Symbol.hasInstance],Object.defineProperty(Writable,Symbol.hasInstance,{value:function value(s){return!!j.call(this,s)||this===Writable&&(s&&s._writableState instanceof WritableState)}})):j=function realHasInstance(s){return s instanceof this},Writable.prototype.pipe=function(){ce(this,new Y)},Writable.prototype.write=function(s,o,i){var a=this._writableState,_=!1,w=!a.objectMode&&function _isUint8Array(s){return x.isBuffer(s)||s instanceof C}(s);return w&&!x.isBuffer(s)&&(s=function _uint8ArrayToBuffer(s){return x.from(s)}(s)),"function"==typeof o&&(i=o,o=null),w?o="buffer":o||(o=a.defaultEncoding),"function"!=typeof i&&(i=nop),a.ending?function writeAfterEnd(s,o){var i=new ie;ce(s,i),u.nextTick(o,i)}(this,i):(w||function validChunk(s,o,i,a){var _;return null===i?_=new ee:"string"==typeof i||o.objectMode||(_=new V("chunk",["string","Buffer"],i)),!_||(ce(s,_),u.nextTick(a,_),!1)}(this,a,s,i))&&(a.pendingcb++,_=function writeOrBuffer(s,o,i,a,u,_){if(!i){var w=function decodeChunk(s,o,i){s.objectMode||!1===s.decodeStrings||"string"!=typeof o||(o=x.from(o,i));return o}(o,a,u);a!==w&&(i=!0,u="buffer",a=w)}var C=o.objectMode?1:a.length;o.length+=C;var j=o.length-1))throw new ae(s);return this._writableState.defaultEncoding=s,this},Object.defineProperty(Writable.prototype,"writableBuffer",{enumerable:!1,get:function get(){return this._writableState&&this._writableState.getBuffer()}}),Object.defineProperty(Writable.prototype,"writableHighWaterMark",{enumerable:!1,get:function get(){return this._writableState.highWaterMark}}),Writable.prototype._write=function(s,o,i){i(new U("_write()"))},Writable.prototype._writev=null,Writable.prototype.end=function(s,o,i){var a=this._writableState;return"function"==typeof s?(i=s,s=null,o=null):"function"==typeof o&&(i=o,o=null),null!=s&&this.write(s,o),a.corked&&(a.corked=1,this.uncork()),a.ending||function endWritable(s,o,i){o.ending=!0,finishMaybe(s,o),i&&(o.finished?u.nextTick(i):s.once("finish",i));o.ended=!0,s.writable=!1}(this,a,i),this},Object.defineProperty(Writable.prototype,"writableLength",{enumerable:!1,get:function get(){return this._writableState.length}}),Object.defineProperty(Writable.prototype,"destroyed",{enumerable:!1,get:function get(){return void 0!==this._writableState&&this._writableState.destroyed},set:function set(s){this._writableState&&(this._writableState.destroyed=s)}}),Writable.prototype.destroy=L.destroy,Writable.prototype._undestroy=L.undestroy,Writable.prototype._destroy=function(s,o){o(s)}},16946:(s,o,i)=>{"use strict";var a=i(1907),u=i(98828),_=i(45807),w=Object,x=a("".split);s.exports=u((function(){return!w("z").propertyIsEnumerable(0)}))?function(s){return"String"===_(s)?x(s,""):w(s)}:w},16962:(s,o)=>{o.aliasToReal={each:"forEach",eachRight:"forEachRight",entries:"toPairs",entriesIn:"toPairsIn",extend:"assignIn",extendAll:"assignInAll",extendAllWith:"assignInAllWith",extendWith:"assignInWith",first:"head",conforms:"conformsTo",matches:"isMatch",property:"get",__:"placeholder",F:"stubFalse",T:"stubTrue",all:"every",allPass:"overEvery",always:"constant",any:"some",anyPass:"overSome",apply:"spread",assoc:"set",assocPath:"set",complement:"negate",compose:"flowRight",contains:"includes",dissoc:"unset",dissocPath:"unset",dropLast:"dropRight",dropLastWhile:"dropRightWhile",equals:"isEqual",identical:"eq",indexBy:"keyBy",init:"initial",invertObj:"invert",juxt:"over",omitAll:"omit",nAry:"ary",path:"get",pathEq:"matchesProperty",pathOr:"getOr",paths:"at",pickAll:"pick",pipe:"flow",pluck:"map",prop:"get",propEq:"matchesProperty",propOr:"getOr",props:"at",symmetricDifference:"xor",symmetricDifferenceBy:"xorBy",symmetricDifferenceWith:"xorWith",takeLast:"takeRight",takeLastWhile:"takeRightWhile",unapply:"rest",unnest:"flatten",useWith:"overArgs",where:"conformsTo",whereEq:"isMatch",zipObj:"zipObject"},o.aryMethod={1:["assignAll","assignInAll","attempt","castArray","ceil","create","curry","curryRight","defaultsAll","defaultsDeepAll","floor","flow","flowRight","fromPairs","invert","iteratee","memoize","method","mergeAll","methodOf","mixin","nthArg","over","overEvery","overSome","rest","reverse","round","runInContext","spread","template","trim","trimEnd","trimStart","uniqueId","words","zipAll"],2:["add","after","ary","assign","assignAllWith","assignIn","assignInAllWith","at","before","bind","bindAll","bindKey","chunk","cloneDeepWith","cloneWith","concat","conformsTo","countBy","curryN","curryRightN","debounce","defaults","defaultsDeep","defaultTo","delay","difference","divide","drop","dropRight","dropRightWhile","dropWhile","endsWith","eq","every","filter","find","findIndex","findKey","findLast","findLastIndex","findLastKey","flatMap","flatMapDeep","flattenDepth","forEach","forEachRight","forIn","forInRight","forOwn","forOwnRight","get","groupBy","gt","gte","has","hasIn","includes","indexOf","intersection","invertBy","invoke","invokeMap","isEqual","isMatch","join","keyBy","lastIndexOf","lt","lte","map","mapKeys","mapValues","matchesProperty","maxBy","meanBy","merge","mergeAllWith","minBy","multiply","nth","omit","omitBy","overArgs","pad","padEnd","padStart","parseInt","partial","partialRight","partition","pick","pickBy","propertyOf","pull","pullAll","pullAt","random","range","rangeRight","rearg","reject","remove","repeat","restFrom","result","sampleSize","some","sortBy","sortedIndex","sortedIndexOf","sortedLastIndex","sortedLastIndexOf","sortedUniqBy","split","spreadFrom","startsWith","subtract","sumBy","take","takeRight","takeRightWhile","takeWhile","tap","throttle","thru","times","trimChars","trimCharsEnd","trimCharsStart","truncate","union","uniqBy","uniqWith","unset","unzipWith","without","wrap","xor","zip","zipObject","zipObjectDeep"],3:["assignInWith","assignWith","clamp","differenceBy","differenceWith","findFrom","findIndexFrom","findLastFrom","findLastIndexFrom","getOr","includesFrom","indexOfFrom","inRange","intersectionBy","intersectionWith","invokeArgs","invokeArgsMap","isEqualWith","isMatchWith","flatMapDepth","lastIndexOfFrom","mergeWith","orderBy","padChars","padCharsEnd","padCharsStart","pullAllBy","pullAllWith","rangeStep","rangeStepRight","reduce","reduceRight","replace","set","slice","sortedIndexBy","sortedLastIndexBy","transform","unionBy","unionWith","update","xorBy","xorWith","zipWith"],4:["fill","setWith","updateWith"]},o.aryRearg={2:[1,0],3:[2,0,1],4:[3,2,0,1]},o.iterateeAry={dropRightWhile:1,dropWhile:1,every:1,filter:1,find:1,findFrom:1,findIndex:1,findIndexFrom:1,findKey:1,findLast:1,findLastFrom:1,findLastIndex:1,findLastIndexFrom:1,findLastKey:1,flatMap:1,flatMapDeep:1,flatMapDepth:1,forEach:1,forEachRight:1,forIn:1,forInRight:1,forOwn:1,forOwnRight:1,map:1,mapKeys:1,mapValues:1,partition:1,reduce:2,reduceRight:2,reject:1,remove:1,some:1,takeRightWhile:1,takeWhile:1,times:1,transform:2},o.iterateeRearg={mapKeys:[1],reduceRight:[1,0]},o.methodRearg={assignInAllWith:[1,0],assignInWith:[1,2,0],assignAllWith:[1,0],assignWith:[1,2,0],differenceBy:[1,2,0],differenceWith:[1,2,0],getOr:[2,1,0],intersectionBy:[1,2,0],intersectionWith:[1,2,0],isEqualWith:[1,2,0],isMatchWith:[2,1,0],mergeAllWith:[1,0],mergeWith:[1,2,0],padChars:[2,1,0],padCharsEnd:[2,1,0],padCharsStart:[2,1,0],pullAllBy:[2,1,0],pullAllWith:[2,1,0],rangeStep:[1,2,0],rangeStepRight:[1,2,0],setWith:[3,1,2,0],sortedIndexBy:[2,1,0],sortedLastIndexBy:[2,1,0],unionBy:[1,2,0],unionWith:[1,2,0],updateWith:[3,1,2,0],xorBy:[1,2,0],xorWith:[1,2,0],zipWith:[1,2,0]},o.methodSpread={assignAll:{start:0},assignAllWith:{start:0},assignInAll:{start:0},assignInAllWith:{start:0},defaultsAll:{start:0},defaultsDeepAll:{start:0},invokeArgs:{start:2},invokeArgsMap:{start:2},mergeAll:{start:0},mergeAllWith:{start:0},partial:{start:1},partialRight:{start:1},without:{start:1},zipAll:{start:0}},o.mutate={array:{fill:!0,pull:!0,pullAll:!0,pullAllBy:!0,pullAllWith:!0,pullAt:!0,remove:!0,reverse:!0},object:{assign:!0,assignAll:!0,assignAllWith:!0,assignIn:!0,assignInAll:!0,assignInAllWith:!0,assignInWith:!0,assignWith:!0,defaults:!0,defaultsAll:!0,defaultsDeep:!0,defaultsDeepAll:!0,merge:!0,mergeAll:!0,mergeAllWith:!0,mergeWith:!0},set:{set:!0,setWith:!0,unset:!0,update:!0,updateWith:!0}},o.realToAlias=function(){var s=Object.prototype.hasOwnProperty,i=o.aliasToReal,a={};for(var u in i){var _=i[u];s.call(a,_)?a[_].push(u):a[_]=[u]}return a}(),o.remap={assignAll:"assign",assignAllWith:"assignWith",assignInAll:"assignIn",assignInAllWith:"assignInWith",curryN:"curry",curryRightN:"curryRight",defaultsAll:"defaults",defaultsDeepAll:"defaultsDeep",findFrom:"find",findIndexFrom:"findIndex",findLastFrom:"findLast",findLastIndexFrom:"findLastIndex",getOr:"get",includesFrom:"includes",indexOfFrom:"indexOf",invokeArgs:"invoke",invokeArgsMap:"invokeMap",lastIndexOfFrom:"lastIndexOf",mergeAll:"merge",mergeAllWith:"mergeWith",padChars:"pad",padCharsEnd:"padEnd",padCharsStart:"padStart",propertyOf:"get",rangeStep:"range",rangeStepRight:"rangeRight",restFrom:"rest",spreadFrom:"spread",trimChars:"trim",trimCharsEnd:"trimEnd",trimCharsStart:"trimStart",zipAll:"zip"},o.skipFixed={castArray:!0,flow:!0,flowRight:!0,iteratee:!0,mixin:!0,rearg:!0,runInContext:!0},o.skipRearg={add:!0,assign:!0,assignIn:!0,bind:!0,bindKey:!0,concat:!0,difference:!0,divide:!0,eq:!0,gt:!0,gte:!0,isEqual:!0,lt:!0,lte:!0,matchesProperty:!0,merge:!0,multiply:!0,overArgs:!0,partial:!0,partialRight:!0,propertyOf:!0,random:!0,range:!0,rangeRight:!0,subtract:!0,zip:!0,zipObject:!0,zipObjectDeep:!0}},17255:(s,o,i)=>{var a=i(47422);s.exports=function basePropertyDeep(s){return function(o){return a(o,s)}}},17285:s=>{function source(s){return s?"string"==typeof s?s:s.source:null}function lookahead(s){return concat("(?=",s,")")}function concat(...s){return s.map((s=>source(s))).join("")}function either(...s){return"("+s.map((s=>source(s))).join("|")+")"}s.exports=function xml(s){const o=concat(/[A-Z_]/,function optional(s){return concat("(",s,")?")}(/[A-Z0-9_.-]*:/),/[A-Z0-9_.-]*/),i={className:"symbol",begin:/&[a-z]+;|&#[0-9]+;|&#x[a-f0-9]+;/},a={begin:/\s/,contains:[{className:"meta-keyword",begin:/#?[a-z_][a-z1-9_-]+/,illegal:/\n/}]},u=s.inherit(a,{begin:/\(/,end:/\)/}),_=s.inherit(s.APOS_STRING_MODE,{className:"meta-string"}),w=s.inherit(s.QUOTE_STRING_MODE,{className:"meta-string"}),x={endsWithParent:!0,illegal:/`]+/}]}]}]};return{name:"HTML, XML",aliases:["html","xhtml","rss","atom","xjb","xsd","xsl","plist","wsf","svg"],case_insensitive:!0,contains:[{className:"meta",begin://,relevance:10,contains:[a,w,_,u,{begin:/\[/,end:/\]/,contains:[{className:"meta",begin://,contains:[a,u,w,_]}]}]},s.COMMENT(//,{relevance:10}),{begin://,relevance:10},i,{className:"meta",begin:/<\?xml/,end:/\?>/,relevance:10},{className:"tag",begin:/)/,end:/>/,keywords:{name:"style"},contains:[x],starts:{end:/<\/style>/,returnEnd:!0,subLanguage:["css","xml"]}},{className:"tag",begin:/)/,end:/>/,keywords:{name:"script"},contains:[x],starts:{end:/<\/script>/,returnEnd:!0,subLanguage:["javascript","handlebars","xml"]}},{className:"tag",begin:/<>|<\/>/},{className:"tag",begin:concat(//,/>/,/\s/)))),end:/\/?>/,contains:[{className:"name",begin:o,relevance:0,starts:x}]},{className:"tag",begin:concat(/<\//,lookahead(concat(o,/>/))),contains:[{className:"name",begin:o,relevance:0},{begin:/>/,relevance:0,endsParent:!0}]}]}}},17400:(s,o,i)=>{var a=i(99374),u=1/0;s.exports=function toFinite(s){return s?(s=a(s))===u||s===-1/0?17976931348623157e292*(s<0?-1:1):s==s?s:0:0===s?s:0}},17533:s=>{s.exports=function yaml(s){var o="true false yes no null",i="[\\w#;/?:@&=+$,.~*'()[\\]]+",a={className:"string",relevance:0,variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/},{begin:/\S+/}],contains:[s.BACKSLASH_ESCAPE,{className:"template-variable",variants:[{begin:/\{\{/,end:/\}\}/},{begin:/%\{/,end:/\}/}]}]},u=s.inherit(a,{variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/},{begin:/[^\s,{}[\]]+/}]}),_={className:"number",begin:"\\b[0-9]{4}(-[0-9][0-9]){0,2}([Tt \\t][0-9][0-9]?(:[0-9][0-9]){2})?(\\.[0-9]*)?([ \\t])*(Z|[-+][0-9][0-9]?(:[0-9][0-9])?)?\\b"},w={end:",",endsWithParent:!0,excludeEnd:!0,keywords:o,relevance:0},x={begin:/\{/,end:/\}/,contains:[w],illegal:"\\n",relevance:0},C={begin:"\\[",end:"\\]",contains:[w],illegal:"\\n",relevance:0},j=[{className:"attr",variants:[{begin:"\\w[\\w :\\/.-]*:(?=[ \t]|$)"},{begin:'"\\w[\\w :\\/.-]*":(?=[ \t]|$)'},{begin:"'\\w[\\w :\\/.-]*':(?=[ \t]|$)"}]},{className:"meta",begin:"^---\\s*$",relevance:10},{className:"string",begin:"[\\|>]([1-9]?[+-])?[ ]*\\n( +)[^ ][^\\n]*\\n(\\2[^\\n]+\\n?)*"},{begin:"<%[%=-]?",end:"[%-]?%>",subLanguage:"ruby",excludeBegin:!0,excludeEnd:!0,relevance:0},{className:"type",begin:"!\\w+!"+i},{className:"type",begin:"!<"+i+">"},{className:"type",begin:"!"+i},{className:"type",begin:"!!"+i},{className:"meta",begin:"&"+s.UNDERSCORE_IDENT_RE+"$"},{className:"meta",begin:"\\*"+s.UNDERSCORE_IDENT_RE+"$"},{className:"bullet",begin:"-(?=[ ]|$)",relevance:0},s.HASH_COMMENT_MODE,{beginKeywords:o,keywords:{literal:o}},_,{className:"number",begin:s.C_NUMBER_RE+"\\b",relevance:0},x,C,a],L=[...j];return L.pop(),L.push(u),w.contains=L,{name:"YAML",case_insensitive:!0,aliases:["yml"],contains:j}}},17670:(s,o,i)=>{var a=i(12651);s.exports=function mapCacheDelete(s){var o=a(this,s).delete(s);return this.size-=o?1:0,o}},17965:(s,o,i)=>{"use strict";var a=i(16426),u={"text/plain":"Text","text/html":"Url",default:"Text"};s.exports=function copy(s,o){var i,_,w,x,C,j,L=!1;o||(o={}),i=o.debug||!1;try{if(w=a(),x=document.createRange(),C=document.getSelection(),(j=document.createElement("span")).textContent=s,j.ariaHidden="true",j.style.all="unset",j.style.position="fixed",j.style.top=0,j.style.clip="rect(0, 0, 0, 0)",j.style.whiteSpace="pre",j.style.webkitUserSelect="text",j.style.MozUserSelect="text",j.style.msUserSelect="text",j.style.userSelect="text",j.addEventListener("copy",(function(a){if(a.stopPropagation(),o.format)if(a.preventDefault(),void 0===a.clipboardData){i&&console.warn("unable to use e.clipboardData"),i&&console.warn("trying IE specific stuff"),window.clipboardData.clearData();var _=u[o.format]||u.default;window.clipboardData.setData(_,s)}else a.clipboardData.clearData(),a.clipboardData.setData(o.format,s);o.onCopy&&(a.preventDefault(),o.onCopy(a.clipboardData))})),document.body.appendChild(j),x.selectNodeContents(j),C.addRange(x),!document.execCommand("copy"))throw new Error("copy command was unsuccessful");L=!0}catch(a){i&&console.error("unable to copy using execCommand: ",a),i&&console.warn("trying IE specific stuff");try{window.clipboardData.setData(o.format||"text",s),o.onCopy&&o.onCopy(window.clipboardData),L=!0}catch(a){i&&console.error("unable to copy using clipboardData: ",a),i&&console.error("falling back to prompt"),_=function format(s){var o=(/mac os x/i.test(navigator.userAgent)?"⌘":"Ctrl")+"+C";return s.replace(/#{\s*key\s*}/g,o)}("message"in o?o.message:"Copy to clipboard: #{key}, Enter"),window.prompt(_,s)}}finally{C&&("function"==typeof C.removeRange?C.removeRange(x):C.removeAllRanges()),j&&document.body.removeChild(j),w()}return L}},18073:(s,o,i)=>{var a=i(85087),u=i(54641),_=i(70981);s.exports=function createRecurry(s,o,i,w,x,C,j,L,B,$){var V=8&o;o|=V?32:64,4&(o&=~(V?64:32))||(o&=-4);var U=[s,o,x,V?C:void 0,V?j:void 0,V?void 0:C,V?void 0:j,L,B,$],z=i.apply(void 0,U);return a(s)&&u(z,U),z.placeholder=w,_(z,s,o)}},19123:(s,o,i)=>{var a=i(65606),u=i(31499),_=i(88310).Stream;function resolve(s,o,i){var a,_=function create_indent(s,o){return new Array(o||0).join(s||"")}(o,i=i||0),w=s;if("object"==typeof s&&((w=s[a=Object.keys(s)[0]])&&w._elem))return w._elem.name=a,w._elem.icount=i,w._elem.indent=o,w._elem.indents=_,w._elem.interrupt=w,w._elem;var x,C=[],j=[];function get_attributes(s){Object.keys(s).forEach((function(o){C.push(function attribute(s,o){return s+'="'+u(o)+'"'}(o,s[o]))}))}switch(typeof w){case"object":if(null===w)break;w._attr&&get_attributes(w._attr),w._cdata&&j.push(("/g,"]]]]>")+"]]>"),w.forEach&&(x=!1,j.push(""),w.forEach((function(s){"object"==typeof s?"_attr"==Object.keys(s)[0]?get_attributes(s._attr):j.push(resolve(s,o,i+1)):(j.pop(),x=!0,j.push(u(s)))})),x||j.push(""));break;default:j.push(u(w))}return{name:a,interrupt:!1,attributes:C,content:j,icount:i,indents:_,indent:o}}function format(s,o,i){if("object"!=typeof o)return s(!1,o);var a=o.interrupt?1:o.content.length;function proceed(){for(;o.content.length;){var u=o.content.shift();if(void 0!==u){if(interrupt(u))return;format(s,u)}}s(!1,(a>1?o.indents:"")+(o.name?"":"")+(o.indent&&!i?"\n":"")),i&&i()}function interrupt(o){return!!o.interrupt&&(o.interrupt.append=s,o.interrupt.end=proceed,o.interrupt=!1,s(!0),!0)}if(s(!1,o.indents+(o.name?"<"+o.name:"")+(o.attributes.length?" "+o.attributes.join(" "):"")+(a?o.name?">":"":o.name?"/>":"")+(o.indent&&a>1?"\n":"")),!a)return s(!1,o.indent?"\n":"");interrupt(o)||proceed()}s.exports=function xml(s,o){"object"!=typeof o&&(o={indent:o});var i=o.stream?new _:null,u="",w=!1,x=o.indent?!0===o.indent?" ":o.indent:"",C=!0;function delay(s){C?a.nextTick(s):s()}function append(s,o){if(void 0!==o&&(u+=o),s&&!w&&(i=i||new _,w=!0),s&&w){var a=u;delay((function(){i.emit("data",a)})),u=""}}function add(s,o){format(append,resolve(s,x,x?1:0),o)}function end(){if(i){var s=u;delay((function(){i.emit("data",s),i.emit("end"),i.readable=!1,i.emit("close")}))}}return delay((function(){C=!1})),o.declaration&&function addXmlDeclaration(s){var o={version:"1.0",encoding:s.encoding||"UTF-8"};s.standalone&&(o.standalone=s.standalone),add({"?xml":{_attr:o}}),u=u.replace("/>","?>")}(o.declaration),s&&s.forEach?s.forEach((function(o,i){var a;i+1===s.length&&(a=end),add(o,a)})):add(s,end),i?(i.readable=!0,i):u},s.exports.element=s.exports.Element=function element(){var s={_elem:resolve(Array.prototype.slice.call(arguments)),push:function(s){if(!this.append)throw new Error("not assigned to a parent!");var o=this,i=this._elem.indent;format(this.append,resolve(s,i,this._elem.icount+(i?1:0)),(function(){o.append(!0)}))},close:function(s){void 0!==s&&this.push(s),this.end&&this.end()}};return s}},19219:s=>{s.exports=function cacheHas(s,o){return s.has(o)}},19287:s=>{"use strict";s.exports={CSSRuleList:0,CSSStyleDeclaration:0,CSSValueList:0,ClientRectList:0,DOMRectList:0,DOMStringList:0,DOMTokenList:1,DataTransferItemList:0,FileList:0,HTMLAllCollection:0,HTMLCollection:0,HTMLFormElement:0,HTMLSelectElement:0,MediaList:0,MimeTypeArray:0,NamedNodeMap:0,NodeList:1,PaintRequestList:0,Plugin:0,PluginArray:0,SVGLengthList:0,SVGNumberList:0,SVGPathSegList:0,SVGPointList:0,SVGStringList:0,SVGTransformList:0,SourceBufferList:0,StyleSheetList:0,TextTrackCueList:0,TextTrackList:0,TouchList:0}},19358:(s,o,i)=>{"use strict";var a=i(85582),u=i(49724),_=i(61626),w=i(88280),x=i(79192),C=i(19595),j=i(54829),L=i(34084),B=i(32096),$=i(39259),V=i(85884),U=i(39447),z=i(7376);s.exports=function(s,o,i,Y){var Z="stackTraceLimit",ee=Y?2:1,ie=s.split("."),ae=ie[ie.length-1],ce=a.apply(null,ie);if(ce){var le=ce.prototype;if(!z&&u(le,"cause")&&delete le.cause,!i)return ce;var pe=a("Error"),de=o((function(s,o){var i=B(Y?o:s,void 0),a=Y?new ce(s):new ce;return void 0!==i&&_(a,"message",i),V(a,de,a.stack,2),this&&w(le,this)&&L(a,this,de),arguments.length>ee&&$(a,arguments[ee]),a}));if(de.prototype=le,"Error"!==ae?x?x(de,pe):C(de,pe,{name:!0}):U&&Z in ce&&(j(de,ce,Z),j(de,ce,"prepareStackTrace")),C(de,ce),!z)try{le.name!==ae&&_(le,"name",ae),le.constructor=de}catch(s){}return de}}},19570:(s,o,i)=>{var a=i(37334),u=i(93243),_=i(83488),w=u?function(s,o){return u(s,"toString",{configurable:!0,enumerable:!1,value:a(o),writable:!0})}:_;s.exports=w},19595:(s,o,i)=>{"use strict";var a=i(49724),u=i(11042),_=i(13846),w=i(74284);s.exports=function(s,o,i){for(var x=u(o),C=w.f,j=_.f,L=0;L{"use strict";var a=i(23034);s.exports=a},19846:(s,o,i)=>{"use strict";var a=i(20798),u=i(98828),_=i(45951).String;s.exports=!!Object.getOwnPropertySymbols&&!u((function(){var s=Symbol("symbol detection");return!_(s)||!(Object(s)instanceof Symbol)||!Symbol.sham&&a&&a<41}))},19931:(s,o,i)=>{var a=i(31769),u=i(68090),_=i(68969),w=i(77797);s.exports=function baseUnset(s,o){return o=a(o,s),null==(s=_(s,o))||delete s[w(u(o))]}},20181:(s,o,i)=>{var a=/^\s+|\s+$/g,u=/^[-+]0x[0-9a-f]+$/i,_=/^0b[01]+$/i,w=/^0o[0-7]+$/i,x=parseInt,C="object"==typeof i.g&&i.g&&i.g.Object===Object&&i.g,j="object"==typeof self&&self&&self.Object===Object&&self,L=C||j||Function("return this")(),B=Object.prototype.toString,$=Math.max,V=Math.min,now=function(){return L.Date.now()};function isObject(s){var o=typeof s;return!!s&&("object"==o||"function"==o)}function toNumber(s){if("number"==typeof s)return s;if(function isSymbol(s){return"symbol"==typeof s||function isObjectLike(s){return!!s&&"object"==typeof s}(s)&&"[object Symbol]"==B.call(s)}(s))return NaN;if(isObject(s)){var o="function"==typeof s.valueOf?s.valueOf():s;s=isObject(o)?o+"":o}if("string"!=typeof s)return 0===s?s:+s;s=s.replace(a,"");var i=_.test(s);return i||w.test(s)?x(s.slice(2),i?2:8):u.test(s)?NaN:+s}s.exports=function debounce(s,o,i){var a,u,_,w,x,C,j=0,L=!1,B=!1,U=!0;if("function"!=typeof s)throw new TypeError("Expected a function");function invokeFunc(o){var i=a,_=u;return a=u=void 0,j=o,w=s.apply(_,i)}function shouldInvoke(s){var i=s-C;return void 0===C||i>=o||i<0||B&&s-j>=_}function timerExpired(){var s=now();if(shouldInvoke(s))return trailingEdge(s);x=setTimeout(timerExpired,function remainingWait(s){var i=o-(s-C);return B?V(i,_-(s-j)):i}(s))}function trailingEdge(s){return x=void 0,U&&a?invokeFunc(s):(a=u=void 0,w)}function debounced(){var s=now(),i=shouldInvoke(s);if(a=arguments,u=this,C=s,i){if(void 0===x)return function leadingEdge(s){return j=s,x=setTimeout(timerExpired,o),L?invokeFunc(s):w}(C);if(B)return x=setTimeout(timerExpired,o),invokeFunc(C)}return void 0===x&&(x=setTimeout(timerExpired,o)),w}return o=toNumber(o)||0,isObject(i)&&(L=!!i.leading,_=(B="maxWait"in i)?$(toNumber(i.maxWait)||0,o):_,U="trailing"in i?!!i.trailing:U),debounced.cancel=function cancel(){void 0!==x&&clearTimeout(x),j=0,a=C=u=x=void 0},debounced.flush=function flush(){return void 0===x?w:trailingEdge(now())},debounced}},20317:s=>{s.exports=function mapToArray(s){var o=-1,i=Array(s.size);return s.forEach((function(s,a){i[++o]=[a,s]})),i}},20334:(s,o,i)=>{"use strict";var a=i(48287).Buffer;class NonError extends Error{constructor(s){super(NonError._prepareSuperMessage(s)),Object.defineProperty(this,"name",{value:"NonError",configurable:!0,writable:!0}),Error.captureStackTrace&&Error.captureStackTrace(this,NonError)}static _prepareSuperMessage(s){try{return JSON.stringify(s)}catch{return String(s)}}}const u=[{property:"name",enumerable:!1},{property:"message",enumerable:!1},{property:"stack",enumerable:!1},{property:"code",enumerable:!0}],_=Symbol(".toJSON called"),destroyCircular=({from:s,seen:o,to_:i,forceEnumerable:w,maxDepth:x,depth:C})=>{const j=i||(Array.isArray(s)?[]:{});if(o.push(s),C>=x)return j;if("function"==typeof s.toJSON&&!0!==s[_])return(s=>{s[_]=!0;const o=s.toJSON();return delete s[_],o})(s);for(const[i,u]of Object.entries(s))"function"==typeof a&&a.isBuffer(u)?j[i]="[object Buffer]":"function"!=typeof u&&(u&&"object"==typeof u?o.includes(s[i])?j[i]="[Circular]":(C++,j[i]=destroyCircular({from:s[i],seen:o.slice(),forceEnumerable:w,maxDepth:x,depth:C})):j[i]=u);for(const{property:o,enumerable:i}of u)"string"==typeof s[o]&&Object.defineProperty(j,o,{value:s[o],enumerable:!!w||i,configurable:!0,writable:!0});return j};s.exports={serializeError:(s,o={})=>{const{maxDepth:i=Number.POSITIVE_INFINITY}=o;return"object"==typeof s&&null!==s?destroyCircular({from:s,seen:[],forceEnumerable:!0,maxDepth:i,depth:0}):"function"==typeof s?`[Function: ${s.name||"anonymous"}]`:s},deserializeError:(s,o={})=>{const{maxDepth:i=Number.POSITIVE_INFINITY}=o;if(s instanceof Error)return s;if("object"==typeof s&&null!==s&&!Array.isArray(s)){const o=new Error;return destroyCircular({from:s,seen:[],to_:o,maxDepth:i,depth:0}),o}return new NonError(s)}}},20426:s=>{var o=Object.prototype.hasOwnProperty;s.exports=function baseHas(s,i){return null!=s&&o.call(s,i)}},20575:(s,o,i)=>{"use strict";var a=i(3121);s.exports=function(s){return a(s.length)}},20798:(s,o,i)=>{"use strict";var a,u,_=i(45951),w=i(96794),x=_.process,C=_.Deno,j=x&&x.versions||C&&C.version,L=j&&j.v8;L&&(u=(a=L.split("."))[0]>0&&a[0]<4?1:+(a[0]+a[1])),!u&&w&&(!(a=w.match(/Edge\/(\d+)/))||a[1]>=74)&&(a=w.match(/Chrome\/(\d+)/))&&(u=+a[1]),s.exports=u},20850:(s,o,i)=>{"use strict";s.exports=i(46076)},20999:(s,o,i)=>{var a=i(69302),u=i(36800);s.exports=function createAssigner(s){return a((function(o,i){var a=-1,_=i.length,w=_>1?i[_-1]:void 0,x=_>2?i[2]:void 0;for(w=s.length>3&&"function"==typeof w?(_--,w):void 0,x&&u(i[0],i[1],x)&&(w=_<3?void 0:w,_=1),o=Object(o);++a<_;){var C=i[a];C&&s(o,C,a,w)}return o}))}},21549:(s,o,i)=>{var a=i(22032),u=i(63862),_=i(66721),w=i(12749),x=i(35749);function Hash(s){var o=-1,i=null==s?0:s.length;for(this.clear();++o{var a=i(16547),u=i(43360);s.exports=function copyObject(s,o,i,_){var w=!i;i||(i={});for(var x=-1,C=o.length;++x{var a=i(51873),u=i(37828),_=i(75288),w=i(25911),x=i(20317),C=i(84247),j=a?a.prototype:void 0,L=j?j.valueOf:void 0;s.exports=function equalByTag(s,o,i,a,j,B,$){switch(i){case"[object DataView]":if(s.byteLength!=o.byteLength||s.byteOffset!=o.byteOffset)return!1;s=s.buffer,o=o.buffer;case"[object ArrayBuffer]":return!(s.byteLength!=o.byteLength||!B(new u(s),new u(o)));case"[object Boolean]":case"[object Date]":case"[object Number]":return _(+s,+o);case"[object Error]":return s.name==o.name&&s.message==o.message;case"[object RegExp]":case"[object String]":return s==o+"";case"[object Map]":var V=x;case"[object Set]":var U=1&a;if(V||(V=C),s.size!=o.size&&!U)return!1;var z=$.get(s);if(z)return z==o;a|=2,$.set(s,o);var Y=w(V(s),V(o),a,j,B,$);return $.delete(s),Y;case"[object Symbol]":if(L)return L.call(s)==L.call(o)}return!1}},22032:(s,o,i)=>{var a=i(81042);s.exports=function hashClear(){this.__data__=a?a(null):{},this.size=0}},22225:s=>{var o="\\ud800-\\udfff",i="\\u2700-\\u27bf",a="a-z\\xdf-\\xf6\\xf8-\\xff",u="A-Z\\xc0-\\xd6\\xd8-\\xde",_="\\xac\\xb1\\xd7\\xf7\\x00-\\x2f\\x3a-\\x40\\x5b-\\x60\\x7b-\\xbf\\u2000-\\u206f \\t\\x0b\\f\\xa0\\ufeff\\n\\r\\u2028\\u2029\\u1680\\u180e\\u2000\\u2001\\u2002\\u2003\\u2004\\u2005\\u2006\\u2007\\u2008\\u2009\\u200a\\u202f\\u205f\\u3000",w="["+_+"]",x="\\d+",C="["+i+"]",j="["+a+"]",L="[^"+o+_+x+i+a+u+"]",B="(?:\\ud83c[\\udde6-\\uddff]){2}",$="[\\ud800-\\udbff][\\udc00-\\udfff]",V="["+u+"]",U="(?:"+j+"|"+L+")",z="(?:"+V+"|"+L+")",Y="(?:['’](?:d|ll|m|re|s|t|ve))?",Z="(?:['’](?:D|LL|M|RE|S|T|VE))?",ee="(?:[\\u0300-\\u036f\\ufe20-\\ufe2f\\u20d0-\\u20ff]|\\ud83c[\\udffb-\\udfff])?",ie="[\\ufe0e\\ufe0f]?",ae=ie+ee+("(?:\\u200d(?:"+["[^"+o+"]",B,$].join("|")+")"+ie+ee+")*"),ce="(?:"+[C,B,$].join("|")+")"+ae,le=RegExp([V+"?"+j+"+"+Y+"(?="+[w,V,"$"].join("|")+")",z+"+"+Z+"(?="+[w,V+U,"$"].join("|")+")",V+"?"+U+"+"+Y,V+"+"+Z,"\\d*(?:1ST|2ND|3RD|(?![123])\\dTH)(?=\\b|[a-z_])","\\d*(?:1st|2nd|3rd|(?![123])\\dth)(?=\\b|[A-Z_])",x,ce].join("|"),"g");s.exports=function unicodeWords(s){return s.match(le)||[]}},22551:(s,o,i)=>{"use strict";var a=i(96540),u=i(69982);function p(s){for(var o="https://reactjs.org/docs/error-decoder.html?invariant="+s,i=1;i