Skip to content

Commit 1bd4dd2

Browse files
FelixKirschJohannes HötterJWittmeyerJWittmeyertianzhou
authored
Release v1.2.0 (#83)
* Update LICENSE * fixes issue #6 (#7) * fixes issue #6 * Adds GitPython requirement to setup script Co-authored-by: JWittmeyer <[email protected]> * Pip install fix (#8) * pypi 1.0.0 -> 1.0.1 * Pip install fix (#10) * cli as part of refinery module * Adds batch change * Update README.md * Add live star-history graph * Update README.md * Update README.md * Adds different credential ip from docker network * Update Readme * Update issue templates adds feature requests * Update issue templates Automatically set label and assignee for enhancements * Update README.md removes discourse and replaces it with GitHub discussions * Adds volume for minio & qdrant for start & template (#17) * update cli to check for commit hash diff * change cli dialogue * Update issue templates * Update issue templates * Changes the windows ipconfig extraction * implements suggestions from PR #32 and bumps version * Adds update script for pulling newest docker images * Adds bat version of the update for windows - changes to docker-compose command * Add files via upload * adds update and help CLI commands, bumps to version 1.1.0 (#74) * V 111 (#75) * adds demo * bumps version * New update logic (#80) * inital version of new update logic * adds checking for updates * add env variable UPDATER for gateway * removes check_for_update on start command of cli * removes branch identifier from clone command in README * adds handling for non running refinery when calling has updates * changes handling of exec envs, changes update logic of cli * changes qdrant version and qdrant local volume * removes wrong indentation * latest changes in .bat scripts * adds checks for exec_envs, has_updates and pip install to .bat scripts * checks docker-compose, changes handling of current branch, adds version update to readme * use curl instead of invoke-webrequest * replace exit with go to end of file * add method to curl request * hide curl output * modifies echo string * replace if by whether because batch forces us * apply changes in update.bat to update script * add sleep to wait for updater-service to start * adds timeout to .bat script * reverts changes in docker-compose template * adds neural search env variable to updater service Co-authored-by: felix0496 <[email protected]> Co-authored-by: Johannes Hötter <[email protected]> Co-authored-by: JWittmeyer <[email protected]> Co-authored-by: JWittmeyer <[email protected]> Co-authored-by: tianzhou <[email protected]> Co-authored-by: wiertzbl <[email protected]> Co-authored-by: simonrempel <[email protected]> Co-authored-by: Simon Degraf <[email protected]> Co-authored-by: felix0496 <[email protected]>
1 parent f66cc80 commit 1bd4dd2

File tree

10 files changed

+212
-56
lines changed

10 files changed

+212
-56
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ refinery/oathkeeper/jwks.json
77
refinery/postgres-data
88
refinery/minio-data
99
refinery/qdrant-data
10+
refinery/qdrant-storage
1011

1112

1213
.DS_Store

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
</p>
66

77
<p align=center>
8-
<a href="https://pypi.org/project/kern-refinery/1.1.1/"><img src="https://img.shields.io/badge/pypi-1.1.1-yellow.svg" alt="pypi 1.1.1"></a>
8+
<a href="https://pypi.org/project/kern-refinery/1.2.0/"><img src="https://img.shields.io/badge/pypi-1.2.0-yellow.svg" alt="pypi 1.2.0"></a>
99
<a href="https://github.com/code-kern-ai/refinery/blob/master/LICENSE"><img src="https://img.shields.io/badge/License-Apache%202.0-success" alt="Apache 2.0 License"></a>
1010
<a href="https://github.com/code-kern-ai/refinery/discussions"><img src="https://img.shields.io/badge/Discussions-gray.svg?logo=github" alt="GitHub Discussions"></a>
1111
<a href="https://discord.gg/qf4rGCEphW"><img src="https://img.shields.io/badge/Discord-gray.svg?logo=discord" alt="Discord"></a>

‎refinery/cli.py‎

Lines changed: 4 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,11 @@
22
import sys
33
import platform
44
import subprocess
5-
import re
65
from git import Repo
76
from wasabi import msg
87

98
REFINERY_REPO = "https://github.com/code-kern-ai/refinery"
109
REFINERY_FOLDER = "refinery"
11-
REMOTE_REPO = "code-kern-ai/refinery.git"
12-
MAIN_BRANCH = "main"
1310

1411

1512
def start(cur_dir: str):
@@ -19,37 +16,7 @@ def start(cur_dir: str):
1916
cur_dir (str): The current directory.
2017
"""
2118

22-
def _start_server(check_for_update: bool):
23-
24-
if check_for_update:
25-
repo = Repo(search_parent_directories=True)
26-
repo_origin = repo.remotes.origin
27-
28-
active_branch = str(repo.active_branch)
29-
repo_identifier_remote = repo_origin.url.split(":")[-1]
30-
31-
if active_branch == MAIN_BRANCH and repo_identifier_remote == REMOTE_REPO:
32-
sha_local = repo.head.object.hexsha
33-
34-
# https://stackoverflow.com/questions/62525382/how-to-get-the-latest-commit-hash-on-remote-using-gitpython
35-
repo_url = f"{REFINERY_REPO}.git"
36-
process = subprocess.Popen(
37-
["git", "ls-remote", repo_url], stdout=subprocess.PIPE
38-
)
39-
stdout, stderr = process.communicate()
40-
sha_remote = re.split(r"\t+", stdout.decode("ascii"))[0]
41-
42-
if sha_local != sha_remote:
43-
msg.info(
44-
"A new version of refinery is available. Should this be pulled? (y/n)"
45-
)
46-
user_input = input("> ")
47-
if user_input == "y":
48-
repo_origin.pull()
49-
msg.good(f"refinery has been updated to commit {sha_remote}.")
50-
else:
51-
msg.info(f"Staying on commit {sha_local}.")
52-
19+
def _start_server():
5320
if platform.system() == "Windows":
5421
subprocess.run(["start.bat"])
5522
else:
@@ -61,12 +28,12 @@ def _start_server(check_for_update: bool):
6128
)
6229
Repo.clone_from(REFINERY_REPO, REFINERY_FOLDER)
6330
with cd(REFINERY_FOLDER):
64-
_start_server(check_for_update=False)
31+
_start_server()
6532
elif cur_dir == REFINERY_FOLDER:
66-
_start_server(check_for_update=True)
33+
_start_server()
6734
else:
6835
with cd(REFINERY_FOLDER):
69-
_start_server(check_for_update=True)
36+
_start_server()
7037

7138

7239
def stop(cur_dir: str):

‎refinery/template/docker-compose.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,7 @@ services:
121121
- NEURAL_SEARCH=http://refinery-neural-search:80
122122
- KRATOS_ADMIN_URL=http://kratos:4434
123123
- WS_NOTIFY_ENDPOINT=http://refinery-websocket:8080
124+
- UPDATER=http://refinery-updater:80
124125
- S3_URI=object-storage:9000 # remove as soon as multipart upload is merged
125126
- S3_ENDPOINT={CRED_ENDPOINT}
126127
- S3_ENDPOINT_LOCAL=object-storage:9000
@@ -147,7 +148,7 @@ services:
147148
- {LOCAL_VOLUME_POSTGRES}:/var/lib/postgresql/data
148149

149150
qdrant:
150-
image: qdrant/qdrant:v0.7.0
151+
image: qdrant/qdrant:v0.9.1
151152
restart: always
152153
ports:
153154
- 6333:6333
@@ -285,6 +286,7 @@ services:
285286
ports:
286287
- 7062:80
287288
environment:
289+
- NEURAL_SEARCH=http://refinery-neural-search:80
288290
- POSTGRES=postgresql://postgres:onetask@graphql-postgres:5432
289291
- S3_ENDPOINT_LOCAL=object-storage:9000
290292
- S3_ACCESS_KEY=onetask

‎setup.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
setup(
1212
name="kern-refinery",
13-
version="1.1.1",
13+
version="1.2.0",
1414
author="jhoetter",
1515
author_email="[email protected]",
1616
description="The open-source data-centric IDE for NLP.",

‎start‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
LOCAL_VOLUME_POSTGRES=./postgres-data
44
LOCAL_VOLUME_MINIO=./minio-data
5-
LOCAL_VOLUME_QDRANT=./qdrant-data
5+
LOCAL_VOLUME_QDRANT=./qdrant-storage
66

77

88
unameOut="$(uname -s)"
@@ -26,10 +26,16 @@ sed -i.bak -e "s|{LOCAL_VOLUME_QDRANT}|$LOCAL_VOLUME_QDRANT|g" refinery/docker-c
2626

2727
rm refinery/*.bak
2828

29-
#pull newest images
30-
docker pull kernai/refinery-lf-exec-env:latest
31-
docker pull kernai/refinery-ml-exec-env:latest
32-
docker pull kernai/refinery-record-ide-env:latest
29+
#pull exec envs if not existent
30+
if [[ "$(docker images -q kernai/refinery-lf-exec-env:latest 2> /dev/null)" == "" ]]; then
31+
docker pull kernai/refinery-lf-exec-env:latest
32+
fi
33+
if [[ "$(docker images -q kernai/refinery-ml-exec-env:latest 2> /dev/null)" == "" ]]; then
34+
docker pull kernai/refinery-ml-exec-env:latest
35+
fi
36+
if [[ "$(docker images -q kernai/refinery-record-ide-env:latest 2> /dev/null)" == "" ]]; then
37+
docker pull kernai/refinery-record-ide-env:latest
38+
fi
3339

3440
if [ ! -f "./refinery/oathkeeper/jwks.json" ]; then
3541
docker run --rm docker.io/oryd/oathkeeper:v0.38 credentials generate --alg RS256 > refinery/oathkeeper/jwks.json

‎start.bat‎

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,12 @@ setlocal enabledelayedexpansion
33

44
set LOCAL_VOLUME_POSTGRES=".\postgres-data"
55
set LOCAL_VOLUME_MINIO=".\minio-data"
6-
set LOCAL_VOLUME_QDRANT=".\qdrant-data"
6+
set LOCAL_VOLUME_QDRANT=".\qdrant-storage"
77

88
rem grab MINIO_ENDPOINT from ipconfig
99
set ip_address_string="IPv4"
1010
for /f "usebackq tokens=2 delims=:" %%f in (`ipconfig ^| findstr /c:%ip_address_string%`) do (
11-
set ip=%%f
11+
set ip=%%f
1212
)
1313

1414
set ip=%ip: =%
@@ -42,22 +42,33 @@ powershell -Command "(gc refinery\docker-compose.yml) -replace '{LOCAL_VOLUME_PO
4242
powershell -Command "(gc refinery\docker-compose.yml) -replace '{LOCAL_VOLUME_MINIO}', '%LOCAL_VOLUME_MINIO%' | Out-File -encoding ASCII refinery\docker-compose.yml"
4343
powershell -Command "(gc refinery\docker-compose.yml) -replace '{LOCAL_VOLUME_QDRANT}', '%LOCAL_VOLUME_QDRANT%' | Out-File -encoding ASCII refinery\docker-compose.yml"
4444

45-
docker pull kernai/refinery-lf-exec-env:latest
46-
docker pull kernai/refinery-ml-exec-env:latest
47-
docker pull kernai/refinery-record-ide-env:latest
45+
for /f "tokens=1" %%i in ('docker images kernai/refinery-lf-exec-env:latest') do (set image=%%i)
46+
if "%image%" neq "kernai/refinery-lf-exec-env" (
47+
docker pull kernai/refinery-lf-exec-env:latest
48+
)
49+
for /f "tokens=1" %%i in ('docker images kernai/refinery-ml-exec-env:latest') do (set image=%%i)
50+
if "%image%" neq "kernai/refinery-ml-exec-env" (
51+
docker pull kernai/refinery-ml-exec-env:latest
52+
)
53+
for /f "tokens=1" %%i in ('docker images kernai/refinery-record-ide-env:latest') do (set image=%%i)
54+
if "%image%" neq "kernai/refinery-record-ide-env" (
55+
docker pull kernai/refinery-record-ide-env:latest
56+
)
4857

4958
IF NOT EXIST .\refinery\oathkeeper\jwks.json (
50-
docker run --rm docker.io/oryd/oathkeeper:v0.38 credentials generate --alg RS256 > refinery\oathkeeper\jwks.json
59+
docker run --rm docker.io/oryd/oathkeeper:v0.38 credentials generate --alg RS256 > refinery\oathkeeper\jwks.json
5160
)
5261

5362

54-
docker-compose -f refinery\docker-compose.yml up -d
63+
docker-compose -f refinery\docker-compose.yml up -d
64+
5565

5666
echo UI: http://localhost:4455/app/
5767
echo Minio: %MINIO_ENDPOINT%
5868
echo MailHog: http://localhost:4436/
5969

60-
pause
70+
71+
if "%1" neq "update" pause
6172
goto :eof
6273

6374

‎stop.bat‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
@echo off
22

3-
docker-compose -f refinery\docker-compose.yml down --remove-orphans
4-
pause
3+
docker-compose -f refinery\docker-compose.yml down --remove-orphans
4+
if "%1" neq "update" pause

‎update‎

Lines changed: 81 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,83 @@
11
#!/bin/bash
22

3-
docker-compose -f refinery/docker-compose.yml pull
3+
echo "Checking refinery installation..."
4+
if test -f "./refinery/docker-compose.yml"; then
5+
echo "Refinery is installed."
6+
else
7+
echo "Refinery is not installed. Run start script to install the latest version."
8+
exit 1
9+
fi
10+
11+
echo "Checking for updates..."
12+
HAS_UPDATES=$(curl -X GET -s 'http://localhost:7062/has_updates?as_html_response=true')
13+
if [ "$HAS_UPDATES" == "True" ]; then
14+
echo "Updates found, updating..."
15+
elif [ "$HAS_UPDATES" == "False" ]; then
16+
echo "No updates available. You are up to date."
17+
exit 0
18+
else
19+
echo "Refinery doesn't seem to run or in a version < 1.2.0. It cannot be checked if any updates are available."
20+
echo "Do you want to try to update anyway? (y/n)"
21+
read -r UPDATE_ANYWAY
22+
if [ "$UPDATE_ANYWAY" == "y" ]; then
23+
echo "Updating..."
24+
else
25+
echo "Exiting..."
26+
exit 0
27+
fi
28+
fi
29+
30+
echo "Stopping running containers..."
31+
source stop
32+
33+
echo "Checking current branch..."
34+
git branch | grep -q "\* release"
35+
if [ $? -eq 0 ]; then
36+
echo "On release branch, pulling latest changes from github..."
37+
git pull
38+
else
39+
echo "The current branch is not the release branch."
40+
echo "Do you want to checkout the release branch and continue updating? (y/n)"
41+
read -r CHECKOUT_RELEASE
42+
if [ "$CHECKOUT_RELEASE" == "y" ]; then
43+
echo "Checking out release branch and pulling latest changes..."
44+
git checkout release
45+
git pull
46+
else
47+
echo "Do you want to continue with the local version? (y/n) "
48+
read -r CONTINUE
49+
if [ "$CONTINUE" == "y" ]; then
50+
echo "Continuing update with the local version..."
51+
else
52+
echo "Stopping the update..."
53+
echo "Exiting..."
54+
exit 0
55+
fi
56+
fi
57+
fi
58+
59+
echo "Checking if kern-refinery is installed..."
60+
pip3 freeze | grep -q "kern-refinery"
61+
if [ $? -eq 0 ]; then
62+
echo "kern-refinery is installed. Updating..."
63+
pip3 install --upgrade kern-refinery
64+
else
65+
echo "kern-refinery is not installed. Skipping update..."
66+
fi
67+
68+
echo "Pulling newest images of exec envs..."
69+
docker pull kernai/refinery-lf-exec-env:latest
70+
docker pull kernai/refinery-ml-exec-env:latest
71+
docker pull kernai/refinery-record-ide-env:latest
72+
73+
echo "Pulling newest images of refinery..."
74+
docker-compose -f refinery/docker-compose.yml pull
75+
76+
echo "Starting refinery containers..."
77+
source start
78+
sleep 10
79+
80+
echo "Triggering refinery-updater to update database..."
81+
curl -X POST http://localhost:7062/update_to_newest > /dev/null 2>&1
82+
83+
echo "Refinery has been updated to the latest version!"

‎update.bat‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,93 @@
11
@echo off
2+
setlocal enabledelayedexpansion
23

4+
echo Checking refinery installation...
5+
if exist "%~dp0\refinery\docker-compose.yml" (
6+
echo Refinery is installed.
7+
) else (
8+
echo Refinery is not installed. Run start.bat to install the latest version.
9+
pause
10+
goto :eof
11+
)
12+
13+
echo Checking for updates...
14+
for /f %%a in (
15+
'curl -X "GET" -s "http://localhost:7062/has_updates?as_html_response=true"'
16+
) do (
17+
set HAS_UPDATES=%%a
18+
)
19+
if "%HAS_UPDATES%" == "True" (
20+
echo Updates found, updating...
21+
) else if "%HAS_UPDATES%" == "False" (
22+
echo No updates available. You are up to date.
23+
pause
24+
goto :eof
25+
) else (
26+
echo Refinery doesn't seem to run or in a version ^< 1.2.0. It cannot be checked if any updates are available.
27+
set /p UPDATE_ANYWAY="Do you want to try to update anyway? (y/n) "
28+
if "!UPDATE_ANYWAY!" == "y" (
29+
echo Updating...
30+
) else (
31+
echo Exiting...
32+
pause
33+
goto :eof
34+
)
35+
)
36+
37+
echo Stopping running containers...
38+
call stop.bat update
39+
40+
41+
echo Checking current branch...
42+
git branch | findstr "^V\* release" > nul
43+
if !ERRORLEVEL! == 0 (
44+
echo On release branch, pulling latest changes from github...
45+
git pull
46+
) else (
47+
echo The current branch is not the release branch.
48+
set /p CHECKOUT_RELEASE="Do you want to checkout the release branch and continue updating? (y/n) "
49+
if "!CHECKOUT_RELEASE!" == "y" (
50+
echo Checking out release branch and pulling latest changes...
51+
git checkout release
52+
git pull
53+
) else (
54+
set /p CONTINUE="Do you want to continue with the local version? (y/n) "
55+
if "!CONTINUE!" == "y" (
56+
echo Continuing update with the local version...
57+
) else (
58+
echo Stopping the update...
59+
echo Exiting...
60+
pause
61+
goto :eof
62+
)
63+
)
64+
)
65+
66+
echo Checking if kern-refinery is installed...
67+
pip3 freeze | findstr "kern-refinery" > nul
68+
if !ERRORLEVEL! == 0 (
69+
echo kern-refinery is installed. Updating...
70+
pip3 install --upgrade kern-refinery
71+
) else (
72+
echo kern-refinery is not installed. Skipping update...
73+
)
74+
75+
echo Pulling newest images of exec envs...
76+
docker pull kernai/refinery-lf-exec-env:latest
77+
docker pull kernai/refinery-ml-exec-env:latest
78+
docker pull kernai/refinery-record-ide-env:latest
79+
80+
echo Pulling newest images of refinery...
381
docker-compose -f refinery\docker-compose.yml pull
82+
83+
84+
echo Starting refinery containers...
85+
call start.bat update
86+
timeout /t 10 /nobreak > nul
87+
88+
echo Triggering refinery-updater to update database...
89+
curl -X "POST" "http://localhost:7062/update_to_newest" > nul
90+
91+
echo Refinery has been updated to the latest version!
92+
493
pause

0 commit comments

Comments
 (0)