Skip to content

Commit 1392c02

Browse files
authored
Merge pull request #213 from aperture-data/release-0.4.2
Release 0.4.2
2 parents f7b59ae + 433e975 commit 1392c02

15 files changed

+294
-103
lines changed

aperturedb/Connector.py

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import sys
3030
import traceback
3131
import os
32+
import requests
3233
import socket
3334
import struct
3435
import time
@@ -84,19 +85,16 @@ class Connector(object):
8485
str (user): Username to specify while establishing a connection.
8586
str (password): Password to specify while connecting to the db.
8687
str (token): Token to use while connecting to the database.
87-
object (session):
8888
bool (use_ssl): Use SSL to encrypt communication with the database.
8989
"""
9090

9191
def __init__(self, host="localhost", port=55555,
9292
user="", password="", token="",
9393
use_ssl=True, shared_data=None):
9494

95-
self.use_ssl = use_ssl
96-
9795
self.host = host
9896
self.port = port
99-
97+
self.use_ssl = use_ssl
10098
self.connected = False
10199
self.last_response = ''
102100
self.last_query_time = 0
@@ -115,7 +113,6 @@ def __init__(self, host="localhost", port=55555,
115113
self.shared_data = shared_data
116114

117115
def __del__(self):
118-
119116
self.conn.close()
120117
self.connected = False
121118

@@ -261,7 +258,7 @@ def _connect(self):
261258
self.connected = True
262259

263260
def _query(self, query, blob_array = []):
264-
261+
response_blob_array = []
265262
# Check the query type
266263
if not isinstance(query, str): # assumes json
267264
query_str = json.dumps(query)
@@ -305,9 +302,12 @@ def _query(self, query, blob_array = []):
305302
logger.warning(
306303
f"Connection broken. Reconnectng attempt [{tries}/3] .. PID = {os.getpid()}")
307304
time.sleep(1)
305+
self.conn.close()
308306
self._connect()
309307
self._renew_session()
310-
308+
if tries == 3:
309+
raise Exception(
310+
f"Could not query apertureDB using TCP.")
311311
return (self.last_response, response_blob_array)
312312

313313
def query(self, q, blobs=[]):
@@ -360,7 +360,11 @@ def _renew_session(self):
360360
count += 1
361361

362362
def create_new_connection(self) -> Connector:
363-
return Connector(self.host, self.port, shared_data=self.shared_data)
363+
return type(self)(
364+
self.host,
365+
self.port,
366+
use_ssl=self.use_ssl,
367+
shared_data=self.shared_data)
364368

365369
def get_last_response_str(self):
366370

aperturedb/ConnectorRest.py

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
#
2+
# The MIT License
3+
#
4+
# @copyright Copyright (c) 2017 Intel Corporation
5+
# @copyright Copyright (c) 2021 ApertureData Inc
6+
#
7+
# Permission is hereby granted, free of charge, to any person obtaining a copy
8+
# of this software and associated documentation files (the "Software"),
9+
# to deal in the Software without restriction,
10+
# including without limitation the rights to use, copy, modify,
11+
# merge, publish, distribute, sublicense, and/or sell
12+
# copies of the Software, and to permit persons to whom the Software is
13+
# furnished to do so, subject to the following conditions:
14+
#
15+
# The above copyright notice and this permission notice shall be included in
16+
# all copies or substantial portions of the Software.
17+
#
18+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
23+
# ARISING FROM,
24+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25+
# THE SOFTWARE.
26+
#
27+
from __future__ import annotations
28+
import os
29+
import requests
30+
import time
31+
import json
32+
import logging
33+
34+
from threading import Lock
35+
from types import SimpleNamespace
36+
from dataclasses import dataclass
37+
from aperturedb.Connector import Connector
38+
39+
logger = logging.getLogger(__name__)
40+
41+
PROTOCOL_VERSION = 1
42+
43+
44+
class UnauthorizedException(Exception):
45+
pass
46+
47+
48+
@dataclass
49+
class Session():
50+
51+
session_token: str
52+
refresh_token: str
53+
session_token_ttl: int
54+
refresh_token_ttl: int
55+
session_started: time.time = time.time()
56+
57+
def valid(self) -> bool:
58+
session_age = time.time() - self.session_started
59+
60+
# This triggers refresh if the session is about to expire.
61+
if session_age > self.session_token_ttl - \
62+
int(os.getenv("SESSION_EXPIRTY_OFFSET_SEC", 10)):
63+
return False
64+
65+
return True
66+
67+
68+
class ConnectorRest(Connector):
69+
"""
70+
.. _connector-label:
71+
72+
**Class to use aperturedb's REST interface**
73+
74+
Args:
75+
str (host): Address of the host to connect to.
76+
int (port): Port to connect to.
77+
str (user): Username to specify while establishing a connection.
78+
str (password): Password to specify while connecting to the db.
79+
str (token): Token to use while connecting to the database.
80+
bool (use_ssl): Use SSL to encrypt communication with the database.
81+
"""
82+
83+
def __init__(self, host="localhost", port=80,
84+
user="", password="", token="",
85+
use_ssl=True, shared_data=None):
86+
87+
self.host = host
88+
self.port = port
89+
self.use_ssl = use_ssl
90+
self.connected = False
91+
92+
self.last_response = ''
93+
self.last_query_time = 0
94+
95+
self.url = ('https' if self.use_ssl else 'http') + \
96+
'://' + host + ':' + str(port) + '/api/'
97+
98+
if shared_data is None:
99+
self.shared_data = SimpleNamespace()
100+
self.shared_data.session = None
101+
self.shared_data.lock = Lock()
102+
try:
103+
self._authenticate(user, password, token)
104+
except Exception as e:
105+
raise Exception("Authentication failed:", str(e))
106+
else:
107+
self.shared_data = shared_data
108+
109+
def __del__(self):
110+
logger.info("Done with connector")
111+
112+
def _query(self, query, blob_array = []):
113+
response_blob_array = []
114+
# Check the query type
115+
if not isinstance(query, str): # assumes json
116+
query_str = json.dumps(query)
117+
else:
118+
query_str = query
119+
120+
files = [
121+
('query', (None, query_str)),
122+
]
123+
124+
for blob in blob_array:
125+
files.append(('blobs', blob))
126+
127+
# Set Auth token, only when not authenticated before
128+
if self.shared_data.session:
129+
headers = {'Authorization': "Bearer " +
130+
self.shared_data.session.session_token}
131+
else:
132+
headers = None
133+
tries = 0
134+
response = SimpleNamespace()
135+
response.status_code = 0
136+
while tries < 3:
137+
tries += 1
138+
response = requests.post(self.url,
139+
headers = headers,
140+
files = files,
141+
verify = self.use_ssl)
142+
if response.status_code == 200:
143+
# Parse response:
144+
json_response = json.loads(response.text)
145+
import base64
146+
response_blob_array = [base64.b64decode(
147+
b) for b in json_response['blobs']]
148+
self.last_response = json_response["json"]
149+
break
150+
logger.error(
151+
f"Response not OK = {response.status_code} {response.text[:1000]}\n\
152+
attempt [{tries}/3] .. PID = {os.getpid()}")
153+
time.sleep(1)
154+
155+
if tries == 3:
156+
raise Exception(
157+
f"Could not query apertureDB using REST.")
158+
return (self.last_response, response_blob_array)

aperturedb/DaskManager.py

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ def __init__(self, num_workers: int = -1):
1919
self.__num_workers = num_workers
2020

2121
def run(self, db: Connector, generator, batchsize, stats):
22-
def process(df, host, port, session):
22+
def process(df, host, port, use_ssl, session, connnector_type):
2323
metrics = Stats()
2424
# Dask reads data in partitions, and the first partition is of 2 rows, with all
2525
# values as 'foo'. This is for sampling the column names and types. Should not process
@@ -32,7 +32,8 @@ def process(df, host, port, session):
3232
shared_data = SimpleNamespace()
3333
shared_data.session = session
3434
shared_data.lock = Lock()
35-
db = Connector(host=host, port=port, shared_data=shared_data)
35+
db = connnector_type(host=host, port=port,
36+
use_ssl=use_ssl, shared_data=shared_data)
3637
except Exception as e:
3738
logger.exception(e)
3839
from aperturedb.ParallelLoader import ParallelLoader
@@ -64,7 +65,9 @@ def process(df, host, port, session):
6465
process,
6566
db.host,
6667
db.port,
67-
db.shared_data.session)
68+
db.use_ssl,
69+
db.shared_data.session,
70+
type(db))
6871
computation = computation.persist()
6972
if stats:
7073
progress(computation)

aperturedb/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77

88
logger = logging.getLogger(__name__)
99

10-
__version__ = "0.4.1"
10+
__version__ = "0.4.2"
1111

1212
# set log level
1313
logger.setLevel(logging.DEBUG)

ci.sh

Lines changed: 46 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,31 @@ set -e
22

33
source $(dirname "$0")/version.sh
44

5+
check_for_changed_docker_files() {
6+
echo "Checking for changed docker files..."
7+
8+
# Get files changed on merge
9+
FILES_CHANGED=$(git diff origin/${TARGET_BRANCH_NAME} origin/${BRANCH_NAME} --name-only | { grep 'Dockerfile' || true; })
10+
11+
echo "Files Changed: " ${FILES_CHANGED}
12+
if [ -z "$FILES_CHANGED" ]
13+
then
14+
echo "No Dockerfile changes."
15+
return
16+
fi
17+
18+
for file in $FILES_CHANGED; do
19+
20+
# Check if dependencies image changed
21+
if [ $file == 'docker/dependencies/Dockerfile' ]
22+
then
23+
DEPENDENCIES_DOCKER_IMAGE_CHANGED=1
24+
echo "Dependencies image changed"
25+
fi
26+
done
27+
echo "Checking for changed docker files...done"
28+
}
29+
530
# Check and updates version based on release branch name
631
update_version() {
732
echo "Checking versions"
@@ -56,6 +81,13 @@ fi
5681
#Install pre requisites
5782
install_prerequisites
5883

84+
echo "Branch: $BRANCH_NAME"
85+
if [ -z "$BRANCH_NAME" ]
86+
then
87+
echo "This is on a merge branch. Will not continue"
88+
exit 0
89+
fi
90+
5991
# Set default version to develop
6092
BUILD_VERSION=develop
6193

@@ -90,7 +122,11 @@ echo "Repository: $DOCKER_REPOSITORY"
90122
build_tests(){
91123
TESTS_IMAGE=$DOCKER_REPOSITORY/aperturedb-python-tests:latest
92124
mkdir -p docker/tests/aperturedata
93-
cp -r aperturedb setup.py README.md test requirements.txt docker/tests/aperturedata
125+
sudo rm -rf test/aperturedb/db
126+
cp -r aperturedb setup.py README.md requirements.txt docker/tests/aperturedata
127+
mkdir -p docker/tests/aperturedata/test/aperturedb
128+
cp -r test/*.py test/*.sh test/input docker/tests/aperturedata/test
129+
cp test/aperturedb/config.json docker/tests/aperturedata/test/aperturedb
94130

95131
echo "Building image $TESTS_IMAGE"
96132
docker build -t $TESTS_IMAGE --cache-from $TESTS_IMAGE -f docker/tests/Dockerfile .
@@ -161,9 +197,17 @@ if [ -z ${ONLY_DEFINES+x} ]
161197
then
162198
if [ -z ${EXCLUDE_TESTING+x} ]
163199
then
200+
check_for_changed_docker_files
201+
echo "DEPENDENCIES_DOCKER_IMAGE_CHANGED=$DEPENDENCIES_DOCKER_IMAGE_CHANGED"
164202
# Dependecies
165203
# TODO : Conditionally build.
166-
build_notebook_dependencies_image
204+
# Check if there is base image change
205+
if [ $DEPENDENCIES_DOCKER_IMAGE_CHANGED == 1 ]
206+
then
207+
build_notebook_dependencies_image
208+
return
209+
fi
210+
167211

168212
# Trigger build notebook image
169213
build_notebook_image

docs/requirements-documentation.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@ sphinx-rtd-theme==1.0.0
66
torch==1.10.2
77
nbconvert==7.0.0
88
ipython==8.0.1
9+
scikit-image==0.19.3

setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
setuptools.setup(
2121
name="aperturedb",
22-
version="0.4.1",
22+
version="0.4.2",
2323
description="ApertureDB Client Module",
2424
install_requires=install_requires,
2525
long_description=long_description,

0 commit comments

Comments
 (0)