Skip to content

Commit 050ae46

Browse files
luisremisluisremis
authored
Merge pull request #16 from aperture-data/release-0.0.10
Release 0.0.10
2 parents 6fe978e + e8b4055 commit 050ae46

33 files changed

+2872
-162
lines changed

‎.github/workflows/main.yaml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# This is a basic workflow to run tests on commit/PRs on develop
2+
3+
name: CI-develop
4+
5+
# Controls when the action will run.
6+
on:
7+
# Triggers the workflow on push or pull request events
8+
# but only for the develop, master, and release branches
9+
push:
10+
branches:
11+
- develop
12+
- master
13+
pull_request:
14+
branches:
15+
- develop
16+
- master
17+
18+
# Allows you to run this workflow manually from the Actions tab
19+
workflow_dispatch:
20+
21+
# A workflow run is made up of one or more jobs
22+
# that can run sequentially or in parallel
23+
jobs:
24+
# This workflow contains a single job called "build-test"
25+
build-test:
26+
# The type of runner that the job will run on Ubuntu 18.04 (latest)
27+
runs-on: ubuntu-latest
28+
29+
# Steps represent a sequence of tasks that will be
30+
# executed as part of the job
31+
steps:
32+
# Checks-out your repository under $GITHUB_WORKSPACE,
33+
# so your job can access it
34+
- uses: actions/checkout@v2
35+
36+
- name: Login to DockerHub
37+
uses: docker/login-action@v1
38+
with:
39+
username: ${{ secrets.DOCKER_USER }}
40+
password: ${{ secrets.DOCKER_PASS }}
41+
42+
- name: Run Tests
43+
run: |
44+
pip3 install .
45+
pip3 install ipython torch torchvision
46+
cd test
47+
bash run_test.sh
48+
49+
- name: Build Notebook Docker
50+
run: |
51+
cd docker
52+
bash build_images.sh

‎.gitignore‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,3 +127,8 @@ dmypy.json
127127

128128
# Pyre type checker
129129
.pyre/
130+
131+
#Data files
132+
*.adb.csv
133+
*.jpg
134+
*.npy

‎README.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# ApertureDB Client Python Module
2+
3+
This is the python client module for ApertureDB.
4+
5+
It provides a connector to AperetureDB instances using
6+
the open source connector for [VDMS](https://github.com/IntelLabs/vdms).
7+
8+
It also implements an Object-Mapper API to interact with
9+
elements in ApertureDB at the object level.
10+
11+
* Status.py provides helper methods to retrieve information about the db.
12+
* Images.py provides the Object-Mapper for image related objetcs (images, bounding boxes, etc)
13+
* NotebookHelpers.py provides helpers to show images/bounding boxes on Jupyter Notebooks
14+
15+
For more information, visit https://aperturedata.io

‎aperturedb/BBoxLoader.py‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
import math
2+
import time
3+
from threading import Thread
4+
5+
import numpy as np
6+
import pandas as pd
7+
8+
from aperturedb import Status
9+
from aperturedb import ParallelLoader
10+
from aperturedb import CSVParser
11+
12+
HEADER_X_POS = "x_pos"
13+
HEADER_Y_POS = "y_pos"
14+
HEADER_WIDTH = "width"
15+
HEADER_HEIGHT = "height"
16+
IMG_KEY_PROP = "img_key_prop"
17+
IMG_KEY_VAL = "img_key_value"
18+
19+
class BBoxGeneratorCSV(CSVParser.CSVParser):
20+
21+
'''
22+
ApertureDB BBox Data loader.
23+
Expects a csv file with the following columns:
24+
25+
IMG_KEY,x_pos,y_pos,width,height,BBOX_PROP_NAME_1, ... BBOX_PROP_NAME_N
26+
27+
IMG_KEY column has the property name of the image property that
28+
the bounding box will be connected to, and each row has the value
29+
that will be used for finding the image.
30+
31+
x_pos,y_pos,width,height are the coordinates of the bounding boxes,
32+
as integers (unit is in pixels)
33+
34+
BBOX_PROP_NAME_N is an arbitrary name of the property of the bounding
35+
box, and each row has the value for that property.
36+
37+
Example csv file:
38+
img_unique_id,x_pos,y_pos,width,height,type
39+
d5b25253-9c1e,257,154,84,125,manual
40+
d5b25253-9c1e,7,537,522,282,manual
41+
...
42+
'''
43+
44+
def __init__(self, filename):
45+
46+
super().__init__(filename)
47+
48+
self.props_keys = [x for x in self.header[5:] if not x.startswith(CSVParser.CONTRAINTS_PREFIX) ]
49+
self.constraints_keys = [x for x in self.header[5:] if x.startswith(CSVParser.CONTRAINTS_PREFIX) ]
50+
51+
self.img_key = self.header[0]
52+
53+
def __getitem__(self, idx):
54+
55+
data = {
56+
"x": int(self.df.loc[idx, HEADER_X_POS]),
57+
"y": int(self.df.loc[idx, HEADER_Y_POS]),
58+
"width": int(self.df.loc[idx, HEADER_WIDTH]),
59+
"height": int(self.df.loc[idx, HEADER_HEIGHT]),
60+
}
61+
62+
val = self.df.loc[idx, self.img_key]
63+
64+
if val != "":
65+
data[IMG_KEY_PROP] = self.img_key
66+
data[IMG_KEY_VAL] = val
67+
68+
properties = self.parse_properties(self.df, idx)
69+
constraints = self.parse_constraints(self.df, idx)
70+
71+
if properties:
72+
data[CSVParser.PROPERTIES] = properties
73+
74+
if constraints:
75+
data[CSVParser.CONSTRAINTS] = constraints
76+
77+
return data
78+
79+
def validate(self):
80+
81+
self.header = list(self.df.columns.values)
82+
83+
if self.header[1] != HEADER_X_POS:
84+
raise Exception("Error with CSV file field: " + HEADER_X_POS)
85+
if self.header[2] != HEADER_Y_POS:
86+
raise Exception("Error with CSV file field: " + HEADER_Y_POS)
87+
if self.header[3] != HEADER_WIDTH:
88+
raise Exception("Error with CSV file field: " + HEADER_WIDTH)
89+
if self.header[4] != HEADER_HEIGHT:
90+
raise Exception("Error with CSV file field: " + HEADER_HEIGHT)
91+
92+
class BBoxLoader(ParallelLoader.ParallelLoader):
93+
94+
def __init__(self, db, dry_run=False):
95+
96+
super().__init__(db, dry_run=dry_run)
97+
98+
self.type = "bbox"
99+
100+
def generate_batch(self, bbox_data):
101+
102+
q = []
103+
104+
ref_counter = 1
105+
for data in bbox_data:
106+
107+
# TODO we could reuse image references within the batch
108+
# instead of creating a new find for every image.
109+
img_ref = ref_counter
110+
ref_counter += 1
111+
fi = {
112+
"FindImage": {
113+
"_ref": img_ref,
114+
}
115+
}
116+
117+
if IMG_KEY_PROP in data:
118+
key = data[IMG_KEY_PROP]
119+
val = data[IMG_KEY_VAL]
120+
constraints = {}
121+
constraints[key] = ["==", val]
122+
fi["FindImage"]["constraints"] = constraints
123+
124+
q.append(fi)
125+
126+
ai = {
127+
"AddBoundingBox": {
128+
"image": img_ref,
129+
"rectangle": {
130+
"x": data["x"],
131+
"y": data["y"],
132+
"width": data["width"],
133+
"height": data["height"],
134+
},
135+
}
136+
}
137+
138+
if "properties" in data:
139+
ai["AddBoundingBox"]["properties"] = data[CSVParser.PROPERTIES]
140+
141+
q.append(ai)
142+
143+
if self.dry_run:
144+
print(q)
145+
146+
return q, []

‎aperturedb/BlobLoader.py‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
import math
2+
import time
3+
from threading import Thread
4+
5+
import numpy as np
6+
import pandas as pd
7+
8+
from aperturedb import Status
9+
from aperturedb import ParallelLoader
10+
from aperturedb import CSVParser
11+
12+
PROPERTIES = "properties"
13+
CONSTRAINTS = "constraints"
14+
BLOB_PATH = "filename"
15+
16+
class BlobGeneratorCSV(CSVParser.CSVParser):
17+
18+
'''
19+
ApertureDB Blob Data loader.
20+
Expects a csv file with the following columns:
21+
22+
filename,PROP_NAME_1, ... PROP_NAME_N,constraint_PROP1
23+
24+
Example csv file:
25+
filename,name,lastname,age,id,constaint_id
26+
/mnt/blob1,John,Salchi,69,321423532,321423532
27+
/mnt/blob2,Johna,Salchi,63,42342522,42342522
28+
...
29+
'''
30+
31+
def __init__(self, filename):
32+
33+
super().__init__(filename)
34+
35+
self.props_keys = [x for x in self.header[1:] if not x.startswith(CSVParser.CONTRAINTS_PREFIX)]
36+
self.props_keys = [x for x in self.props_keys if x != BLOB_PATH]
37+
self.constraints_keys = [x for x in self.header[1:] if x.startswith(CSVParser.CONTRAINTS_PREFIX) ]
38+
39+
def __getitem__(self, idx):
40+
41+
data = {}
42+
43+
filename = self.df.loc[idx, BLOB_PATH]
44+
blob_ok, blob = self.load_blob(filename)
45+
if not blob_ok:
46+
Exception("Error loading blob: " + filename )
47+
data["blob"] = blob
48+
49+
properties = self.parse_properties (self.df, idx)
50+
constraints = self.parse_constraints(self.df, idx)
51+
52+
if properties:
53+
data[PROPERTIES] = properties
54+
55+
if constraints:
56+
data[CONSTRAINTS] = constraints
57+
58+
return data
59+
60+
def load_blob(self, filename):
61+
62+
try:
63+
fd = open(filename, "rb")
64+
buff = fd.read()
65+
fd.close()
66+
return True, buff
67+
except:
68+
print("BLOB ERROR:", filename)
69+
70+
return False, None
71+
72+
def validate(self):
73+
74+
self.header = list(self.df.columns.values)
75+
76+
if self.header[0] != BLOB_PATH:
77+
raise Exception("Error with CSV file field: " + BLOB_PATH)
78+
79+
class BlobLoader(ParallelLoader.ParallelLoader):
80+
81+
'''
82+
ApertureDB Blob Loader.
83+
84+
This class is to be used in combination with a "generator".
85+
The generator must be an iterable object that generated "Blob_data"
86+
elements:
87+
Blob_data = {
88+
"properties": properties,
89+
"constraints": constraints,
90+
}
91+
'''
92+
93+
def __init__(self, db, dry_run=False):
94+
95+
super().__init__(db, dry_run=dry_run)
96+
97+
self.type = "blob"
98+
99+
def generate_batch(self, Blob_data):
100+
101+
q = []
102+
blobs = []
103+
104+
for data in Blob_data:
105+
106+
ae = {
107+
"AddBlob": {
108+
}
109+
}
110+
111+
if PROPERTIES in data:
112+
ae["AddBlob"][PROPERTIES] = data[PROPERTIES]
113+
114+
if CONSTRAINTS in data:
115+
ae["AddBlob"]["if_not_found"] = data[CONSTRAINTS]
116+
117+
q.append(ae)
118+
119+
if len(data["blob"]) == 0:
120+
print("WARNING: Skipping empty blob.")
121+
continue
122+
blobs.append(data["blob"])
123+
124+
if self.dry_run:
125+
print(q)
126+
127+
return q, blobs

0 commit comments

Comments
 (0)