Skip to content

Commit 7c063e2

Browse files
akhileshhakhileshh
committed
migrate to the new kvdbclient
1 parent 0703be7 commit 7c063e2

File tree

12 files changed

+111
-47
lines changed

12 files changed

+111
-47
lines changed

‎Dockerfile‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,4 @@ ENV GIT_SSL_NO_VERIFY=1
55
RUN mkdir -p /home/nginx/.cloudvolume/secrets && chown -R nginx /home/nginx && usermod -d /home/nginx -s /bin/bash nginx
66

77
COPY . /app
8-
RUN pip install --no-cache-dir --upgrade -r requirements.txt \
9-
&& pip install --upgrade git+https://github.com/seung-lab/KVDbClient.git@main
8+
RUN pip install --no-cache-dir --upgrade -r requirements.txt

‎cloudbuild.v1.ingest.yaml‎

Lines changed: 30 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,32 @@
11
steps:
2-
- name: "gcr.io/cloud-builders/docker"
3-
timeout: 600s
4-
args:
5-
- build
6-
- "--tag=gcr.io/$PROJECT_ID/pcgl2cache:ingest_v1"
7-
- "--file=./ingest.v1.Dockerfile"
8-
- .
2+
- name: "gcr.io/cloud-builders/docker"
3+
entrypoint: "bash"
4+
args: ["-c", "docker login --username=$$USERNAME --password=$$PASSWORD"]
5+
secretEnv: ["USERNAME", "PASSWORD"]
6+
- name: "gcr.io/cloud-builders/docker"
7+
timeout: 600s
8+
args:
9+
- build
10+
- "--tag=gcr.io/$PROJECT_ID/pcgl2cache:ingest_v1"
11+
- "--file=./ingest.v1.Dockerfile"
12+
- .
13+
- name: "gcr.io/cloud-builders/docker"
14+
entrypoint: "bash"
15+
args:
16+
[
17+
"-c",
18+
"docker tag gcr.io/$PROJECT_ID/pcgl2cache:ingest_v1 $$USERNAME/pcgl2cache:ingest_v1",
19+
]
20+
secretEnv: ["USERNAME"]
21+
- name: "gcr.io/cloud-builders/docker"
22+
entrypoint: "bash"
23+
args: ["-c", "docker push $$USERNAME/pcgl2cache:ingest_v1"]
24+
secretEnv: ["USERNAME"]
925
images:
10-
- "gcr.io/$PROJECT_ID/pcgl2cache:ingest_v1"
26+
- "gcr.io/$PROJECT_ID/pcgl2cache:ingest_v1"
27+
availableSecrets:
28+
secretManager:
29+
- versionName: projects/$PROJECT_ID/secrets/docker-password/versions/1
30+
env: "PASSWORD"
31+
- versionName: projects/$PROJECT_ID/secrets/docker-username/versions/1
32+
env: "USERNAME"

‎cloudbuild.v1.worker.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ steps:
77
timeout: 600s
88
args:
99
- build
10-
- "--tag=gcr.io/neuromancer-seung-import/pcgl2cache:worker_pcgv1_$TAG_NAME"
10+
- "--tag=gcr.io/$PROJECT_ID/pcgl2cache:worker_pcgv1_$TAG_NAME"
1111
- "--file=./ingest.v1.Dockerfile"
1212
- .
1313
- name: "gcr.io/cloud-builders/docker"
@@ -23,7 +23,7 @@ steps:
2323
args: ["-c", "docker push $$USERNAME/pcgl2cache:worker_pcgv1_$TAG_NAME"]
2424
secretEnv: ["USERNAME"]
2525
images:
26-
- "gcr.io/neuromancer-seung-import/pcgl2cache:worker_pcgv1_$TAG_NAME"
26+
- "gcr.io/$PROJECT_ID/pcgl2cache:worker_pcgv1_$TAG_NAME"
2727
availableSecrets:
2828
secretManager:
2929
- versionName: projects/$PROJECT_ID/secrets/docker-password/versions/1

‎cloudbuild.v2.ingest.yaml‎

Lines changed: 30 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,32 @@
11
steps:
2-
- name: "gcr.io/cloud-builders/docker"
3-
timeout: 600s
4-
args:
5-
- build
6-
- "--tag=gcr.io/$PROJECT_ID/pcgl2cache:ingest_v2"
7-
- "--file=./ingest.v2.Dockerfile"
8-
- .
2+
- name: "gcr.io/cloud-builders/docker"
3+
entrypoint: "bash"
4+
args: ["-c", "docker login --username=$$USERNAME --password=$$PASSWORD"]
5+
secretEnv: ["USERNAME", "PASSWORD"]
6+
- name: "gcr.io/cloud-builders/docker"
7+
timeout: 600s
8+
args:
9+
- build
10+
- "--tag=gcr.io/$PROJECT_ID/pcgl2cache:ingest_v2"
11+
- "--file=./ingest.v2.Dockerfile"
12+
- .
13+
- name: "gcr.io/cloud-builders/docker"
14+
entrypoint: "bash"
15+
args:
16+
[
17+
"-c",
18+
"docker tag gcr.io/$PROJECT_ID/pcgl2cache:ingest_v2 $$USERNAME/pcgl2cache:ingest_v2",
19+
]
20+
secretEnv: ["USERNAME"]
21+
- name: "gcr.io/cloud-builders/docker"
22+
entrypoint: "bash"
23+
args: ["-c", "docker push $$USERNAME/pcgl2cache:ingest_v2"]
24+
secretEnv: ["USERNAME"]
925
images:
10-
- "gcr.io/$PROJECT_ID/pcgl2cache:ingest_v2"
26+
- "gcr.io/$PROJECT_ID/pcgl2cache:ingest_v2"
27+
availableSecrets:
28+
secretManager:
29+
- versionName: projects/$PROJECT_ID/secrets/docker-password/versions/1
30+
env: "PASSWORD"
31+
- versionName: projects/$PROJECT_ID/secrets/docker-username/versions/1
32+
env: "USERNAME"

‎cloudbuild.v2.worker.yaml‎

Lines changed: 30 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,32 @@
11
steps:
2-
- name: "gcr.io/cloud-builders/docker"
3-
timeout: 600s
4-
args:
5-
- build
6-
- "--tag=gcr.io/$PROJECT_ID/pcgl2cache:worker_pcgv2_$TAG_NAME"
7-
- "--file=./ingest.v2.Dockerfile"
8-
- .
2+
- name: "gcr.io/cloud-builders/docker"
3+
entrypoint: "bash"
4+
args: ["-c", "docker login --username=$$USERNAME --password=$$PASSWORD"]
5+
secretEnv: ["USERNAME", "PASSWORD"]
6+
- name: "gcr.io/cloud-builders/docker"
7+
timeout: 600s
8+
args:
9+
- build
10+
- "--tag=gcr.io/$PROJECT_ID/pcgl2cache:worker_pcgv2_$TAG_NAME"
11+
- "--file=./ingest.v2.Dockerfile"
12+
- .
13+
- name: "gcr.io/cloud-builders/docker"
14+
entrypoint: "bash"
15+
args:
16+
[
17+
"-c",
18+
"docker tag gcr.io/$PROJECT_ID/pcgl2cache:worker_pcgv2_$TAG_NAME $$USERNAME/pcgl2cache:worker_pcgv2_$TAG_NAME",
19+
]
20+
secretEnv: ["USERNAME"]
21+
- name: "gcr.io/cloud-builders/docker"
22+
entrypoint: "bash"
23+
args: ["-c", "docker push $$USERNAME/pcgl2cache:worker_pcgv2_$TAG_NAME"]
24+
secretEnv: ["USERNAME"]
925
images:
10-
- "gcr.io/$PROJECT_ID/pcgl2cache:worker_pcgv2_$TAG_NAME"
26+
- "gcr.io/$PROJECT_ID/pcgl2cache:worker_pcgv2_$TAG_NAME"
27+
availableSecrets:
28+
secretManager:
29+
- versionName: projects/$PROJECT_ID/secrets/docker-password/versions/1
30+
env: "PASSWORD"
31+
- versionName: projects/$PROJECT_ID/secrets/docker-username/versions/1
32+
env: "USERNAME"

‎ingest.v1.Dockerfile‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,4 @@ RUN mkdir -p /home/nginx/.cloudvolume/secrets && chown -R nginx /home/nginx && u
55

66
COPY . /app
77
RUN pip install --no-cache-dir --upgrade -r requirements.txt \
8-
&& pip install --upgrade git+https://github.com/CAVEconnectome/PyChunkedGraph.git@pcgv1 \
9-
&& pip install --upgrade git+https://github.com/seung-lab/KVDbClient.git@main
8+
&& pip install --upgrade git+https://github.com/CAVEconnectome/PyChunkedGraph.git@pcgv1

‎ingest.v2.Dockerfile‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ RUN mkdir -p /home/nginx/.cloudvolume/secrets && chown -R nginx /home/nginx && u
88

99
COPY requirements.txt .
1010
RUN pip install --upgrade --no-cache-dir -r requirements.txt \
11-
&& pip install --upgrade git+https://github.com/CAVEconnectome/PyChunkedGraph.git@main \
12-
&& pip install --upgrade git+https://github.com/seung-lab/KVDbClient.git@main
11+
&& pip install --upgrade git+https://github.com/CAVEconnectome/PyChunkedGraph.git@main
1312

1413
COPY . /app

‎pcgl2cache/core/attributes.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import numpy as np
22
from kvdbclient.serializers import NumPyArray
33
from kvdbclient.serializers import NumPyValue
4-
from kvdbclient.bigtable.attributes import Attribute
4+
from kvdbclient.attributes import _Attribute as Attribute
55

66
UINT64 = np.dtype("uint64").newbyteorder("L")
77
UINT32 = np.dtype("uint32").newbyteorder("L")

‎pcgl2cache/core/features.py‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
from edt import edt
77
from sklearn import decomposition
88
from kvdbclient import BigTableClient
9+
from kvdbclient.base import serialize_uint64
910
from cloudvolume import CloudVolume
1011

1112
from . import attributes
@@ -314,10 +315,7 @@ def run_l2cache(
314315

315316

316317
def write_to_db(client: BigTableClient, result_d: dict) -> None:
317-
from kvdbclient.base import Entry
318-
from kvdbclient.base import EntryKey
319-
320-
entries = []
318+
rows = []
321319
for tup in zip(*result_d.values()):
322320
(
323321
l2id,
@@ -340,5 +338,5 @@ def write_to_db(client: BigTableClient, result_d: dict) -> None:
340338
attributes.PCA: pca_comp,
341339
attributes.PCA_VAL: pca_vals,
342340
}
343-
entries.append(Entry(EntryKey(l2id), val_d))
344-
client.write_entries(entries)
341+
rows.append(client.mutate_row(serialize_uint64(l2id), val_d))
342+
client.write(rows)

‎pcgl2cache/ingest/cli.py‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import click
66
from flask.cli import AppGroup
77

8+
import pcgl2cache
89
from .manager import IngestionManager
910
from .redis import get_redis_connection
1011

@@ -43,8 +44,9 @@ def ingest_cache(
4344
from kvdbclient import BigTableClient
4445
from kvdbclient import get_default_client_info
4546

46-
client = BigTableClient(config=get_default_client_info().CONFIG)
47-
client.create_table(cache_id)
47+
meta = {"graph_id": graph_id, "cv_path": cv_path, "timestamp": timestamp}
48+
client = BigTableClient(cache_id, config=get_default_client_info().CONFIG)
49+
client.create_table(meta=meta, version=pcgl2cache.__version__)
4850

4951
# example format 2018-06-29 08:15:27
5052
timestamp = datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S").replace(
@@ -86,8 +88,9 @@ def ingest_cache_v2(
8688
from kvdbclient import BigTableClient
8789
from kvdbclient import get_default_client_info
8890

89-
client = BigTableClient(config=get_default_client_info().CONFIG)
90-
client.create_table(cache_id)
91+
meta = {"graph_id": graph_id, "cv_path": cv_path, "timestamp": timestamp}
92+
client = BigTableClient(cache_id, config=get_default_client_info().CONFIG)
93+
client.create_table(meta=meta, version=pcgl2cache.__version__)
9194

9295
# example format 2022-06-29 08:15:27
9396
timestamp = datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S").replace(

0 commit comments

Comments
 (0)