Skip to content

Commit ebb2fad

Browse files
claffinclaffin
authored
Merge pull request #91 from Calvin-Huang/fix/gcp
Fix GCP provider
2 parents 7f5f6e8 + fe2b7cb commit ebb2fad

File tree

5 files changed

+195
-45
lines changed

5 files changed

+195
-45
lines changed

‎cloudproxy/check.py‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,22 @@ def fetch_ip(ip_address):
5151

5252
def check_alive(ip_address):
5353
try:
54-
result = requests.get("http://ipecho.net/plain", proxies={'http': "http://" + ip_address + ":8899"}, timeout=10)
54+
if settings.config["no_auth"]:
55+
proxies = {
56+
"http": "http://" + ip_address + ":8899",
57+
"https": "http://" + ip_address + ":8899",
58+
}
59+
else:
60+
auth = (
61+
settings.config["auth"]["username"] + ":" + settings.config["auth"]["password"]
62+
)
63+
64+
proxies = {
65+
"http": "http://" + auth + "@" + ip_address + ":8899",
66+
"https": "http://" + auth + "@" + ip_address + ":8899",
67+
}
68+
69+
result = requests.get("http://ipecho.net/plain", proxies=proxies, timeout=10)
5570
if result.status_code in (200, 407):
5671
return True
5772
else:

‎cloudproxy/providers/gcp/functions.py‎

Lines changed: 106 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -9,28 +9,66 @@
99
from cloudproxy.providers.config import set_auth
1010
from cloudproxy.providers.settings import config
1111

12-
gcp = config["providers"]["gcp"]
13-
if gcp["enabled"] == 'True':
12+
gcp = None
13+
compute = None
14+
15+
def get_client(instance_config=None):
16+
"""
17+
Initialize and return a GCP client based on the provided configuration.
18+
19+
Args:
20+
instance_config: The specific instance configuration
21+
22+
Returns:
23+
tuple: (config, gcp_client)
24+
"""
25+
26+
global gcp, compute
27+
if gcp is not None and compute is not None:
28+
return gcp, compute
29+
30+
if instance_config is None:
31+
instance_config = config["providers"]["gcp"]["instances"]["default"]
32+
33+
gcp = config["providers"]["gcp"]
1434
try:
15-
credentials = service_account.Credentials.from_service_account_info(
16-
json.loads(gcp["secrets"]["service_account_key"])
17-
)
35+
if 'sa_json' in instance_config["secrets"]:
36+
credentials = service_account.Credentials.from_service_account_file(
37+
instance_config["secrets"]["sa_json"]
38+
)
39+
else:
40+
credentials = service_account.Credentials.from_service_account_info(
41+
json.loads(instance_config["secrets"]["service_account_key"])
42+
)
1843
compute = googleapiclient.discovery.build('compute', 'v1', credentials=credentials)
44+
45+
return gcp, compute
1946
except TypeError:
2047
logger.error("GCP -> Invalid service account key")
2148

2249

23-
def create_proxy():
50+
def create_proxy(instance_config=None):
51+
"""
52+
Create a GCP proxy instance.
53+
54+
Args:
55+
instance_config: The specific instance configuration
56+
"""
57+
if instance_config is None:
58+
instance_config = config["providers"]["gcp"]["instances"]["default"]
59+
60+
gcp, compute = get_client(instance_config)
61+
2462
image_response = compute.images().getFromFamily(
25-
project=gcp["image_project"],
26-
family=gcp["image_family"]
63+
project=instance_config["image_project"],
64+
family=instance_config["image_family"]
2765
).execute()
2866
source_disk_image = image_response['selfLink']
2967

3068
body = {
3169
'name': 'cloudproxy-' + str(uuid.uuid4()),
3270
'machineType':
33-
f"zones/{gcp['zone']}/machineTypes/{gcp['size']}",
71+
f"zones/{instance_config['zone']}/machineTypes/{instance_config['size']}",
3472
'tags': {
3573
'items': [
3674
'cloudproxy'
@@ -67,48 +105,93 @@ def create_proxy():
67105
}
68106

69107
return compute.instances().insert(
70-
project=gcp["project"],
71-
zone=gcp["zone"],
108+
project=instance_config["project"],
109+
zone=instance_config["zone"],
72110
body=body
73111
).execute()
74112

75-
def delete_proxy(name):
113+
def delete_proxy(name, instance_config=None):
114+
"""
115+
Delete a GCP proxy instance.
116+
117+
Args:
118+
name: Name of the instance to delete
119+
instance_config: The specific instance configuration
120+
"""
121+
if instance_config is None:
122+
instance_config = config["providers"]["gcp"]["instances"]["default"]
123+
124+
gcp, compute = get_client(instance_config)
125+
76126
try:
77127
return compute.instances().delete(
78-
project=gcp["project"],
79-
zone=gcp["zone"],
128+
project=instance_config["project"],
129+
zone=instance_config["zone"],
80130
instance=name
81131
).execute()
82132
except googleapiclient.errors.HttpError:
83133
logger.info(f"GCP --> HTTP Error when trying to delete proxy {name}. Probably has already been deleted.")
84134
return None
85135

86-
def stop_proxy(name):
136+
def stop_proxy(name, instance_config=None):
137+
"""
138+
Stop a GCP proxy instance.
139+
140+
Args:
141+
name: Name of the instance to stop
142+
instance_config: The specific instance configuration
143+
"""
144+
if instance_config is None:
145+
instance_config = config["providers"]["gcp"]["instances"]["default"]
146+
87147
try:
88148
return compute.instances().stop(
89-
project=gcp["project"],
90-
zone=gcp["zone"],
149+
project=instance_config["project"],
150+
zone=instance_config["zone"],
91151
instance=name
92152
).execute()
93153
except googleapiclient.errors.HttpError:
94154
logger.info(f"GCP --> HTTP Error when trying to stop proxy {name}. Probably has already been deleted.")
95155
return None
96156

97-
def start_proxy(name):
157+
def start_proxy(name, instance_config=None):
158+
"""
159+
Start a GCP proxy instance.
160+
161+
Args:
162+
name: Name of the instance to start
163+
instance_config: The specific instance configuration
164+
"""
165+
if instance_config is None:
166+
instance_config = config["providers"]["gcp"]["instances"]["default"]
167+
168+
gcp, compute = get_client(instance_config)
169+
98170
try:
99171
return compute.instances().start(
100-
project=gcp["project"],
101-
zone=gcp["zone"],
172+
project=instance_config["project"],
173+
zone=instance_config["zone"],
102174
instance=name
103175
).execute()
104176
except googleapiclient.errors.HttpError:
105177
logger.info(f"GCP --> HTTP Error when trying to start proxy {name}. Probably has already been deleted.")
106178
return None
107179

108-
def list_instances():
180+
def list_instances(instance_config=None):
181+
"""
182+
List all GCP proxy instances.
183+
184+
Args:
185+
instance_config: The specific instance configuration
186+
"""
187+
if instance_config is None:
188+
instance_config = config["providers"]["gcp"]["instances"]["default"]
189+
190+
gcp, compute = get_client(instance_config)
191+
109192
result = compute.instances().list(
110-
project=gcp["project"],
111-
zone=gcp["zone"],
193+
project=instance_config["project"],
194+
zone=instance_config["zone"],
112195
filter='labels.cloudproxy eq cloudproxy'
113196
).execute()
114197
return result['items'] if 'items' in result else []

‎cloudproxy/providers/gcp/main.py‎

Lines changed: 68 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -13,30 +13,49 @@
1313
)
1414
from cloudproxy.providers.settings import delete_queue, restart_queue, config
1515

16-
def gcp_deployment(min_scaling):
17-
total_instances = len(list_instances())
16+
def gcp_deployment(min_scaling, instance_config=None):
17+
"""
18+
Deploy GCP instances based on min_scaling requirements.
19+
20+
Args:
21+
min_scaling: The minimum number of instances to maintain
22+
instance_config: The specific instance configuration
23+
"""
24+
if instance_config is None:
25+
instance_config = config["providers"]["gcp"]["instances"]["default"]
26+
27+
total_instances = len(list_instances(instance_config))
1828
if min_scaling < total_instances:
1929
logger.info("Overprovisioned: GCP destroying.....")
2030
for instance in itertools.islice(
21-
list_instances(), 0, (total_instances - min_scaling)
31+
list_instances(instance_config), 0, (total_instances - min_scaling)
2232
):
2333
access_configs = instance['networkInterfaces'][0]['accessConfigs'][0]
2434
msg = f"{instance['name']} {access_configs['natIP']}"
25-
delete_proxy(instance['name'])
35+
delete_proxy(instance['name'], instance_config)
2636
logger.info("Destroyed: GCP -> " + msg)
2737
if min_scaling - total_instances < 1:
2838
logger.info("Minimum GCP instances met")
2939
else:
3040
total_deploy = min_scaling - total_instances
3141
logger.info("Deploying: " + str(total_deploy) + " GCP instances")
3242
for _ in range(total_deploy):
33-
create_proxy()
43+
create_proxy(instance_config)
3444
logger.info("Deployed")
35-
return len(list_instances())
45+
return len(list_instances(instance_config))
46+
47+
def gcp_check_alive(instance_config=None):
48+
"""
49+
Check if any GCP instances are alive.
50+
51+
Args:
52+
instance_config: The specific instance configuration
53+
"""
54+
if instance_config is None:
55+
instance_config = config["providers"]["gcp"]["instances"]["default"]
3656

37-
def gcp_check_alive():
3857
ip_ready = []
39-
for instance in list_instances():
58+
for instance in list_instances(instance_config):
4059
try:
4160
elapsed = datetime.datetime.now(
4261
datetime.timezone.utc
@@ -50,7 +69,7 @@ def gcp_check_alive():
5069

5170
elif instance['status'] == "TERMINATED":
5271
logger.info("Waking up: GCP -> Instance " + instance['name'])
53-
started = start_proxy(instance['name'])
72+
started = start_proxy(instance['name'], instance_config)
5473
if not started:
5574
logger.info("Could not wake up, trying again later.")
5675

@@ -83,27 +102,57 @@ def gcp_check_alive():
83102
logger.info("Pending: GCP -> Allocating IP")
84103
return ip_ready
85104

86-
def gcp_check_delete():
87-
for instance in list_instances():
105+
def gcp_check_delete(instance_config=None):
106+
"""
107+
Check if any GCP instances need to be deleted.
108+
109+
Args:
110+
instance_config: The specific instance configuration
111+
"""
112+
if instance_config is None:
113+
instance_config = config["providers"]["gcp"]["instances"]["default"]
114+
115+
for instance in list_instances(instance_config):
88116
access_configs = instance['networkInterfaces'][0]['accessConfigs'][0]
89117
if 'natIP' in access_configs and access_configs['natIP'] in delete_queue:
90118
msg = f"{instance['name']}, {access_configs['natIP']}"
91119
delete_proxy(instance['name'])
92120
logger.info("Destroyed: not wanted -> " + msg)
93121
delete_queue.remove(access_configs['natIP'])
94122

95-
def gcp_check_stop():
96-
for instance in list_instances():
123+
def gcp_check_stop(instance_config=None):
124+
"""
125+
Check if any GCP instances need to be stopped.
126+
127+
Args:
128+
instance_config: The specific instance configuration
129+
"""
130+
if instance_config is None:
131+
instance_config = config["providers"]["gcp"]["instances"]["default"]
132+
133+
for instance in list_instances(instance_config):
97134
access_configs = instance['networkInterfaces'][0]['accessConfigs'][0]
98135
if 'natIP' in access_configs and access_configs['natIP'] in restart_queue:
99136
msg = f"{instance['name']}, {access_configs['natIP']}"
100-
stop_proxy(instance['name'])
137+
stop_proxy(instance['name'], instance_config)
101138
logger.info("Stopped: getting new IP -> " + msg)
102139
restart_queue.remove(access_configs['natIP'])
103140

104-
def gcp_start():
105-
gcp_check_delete()
106-
gcp_check_stop()
107-
gcp_deployment(config["providers"]["gcp"]["scaling"]["min_scaling"])
108-
ip_ready = gcp_check_alive()
141+
def gcp_start(instance_config=None):
142+
"""
143+
Start the GCP provider lifecycle.
144+
145+
Args:
146+
instance_config: The specific instance configuration
147+
148+
Returns:
149+
list: List of ready IP addresses
150+
"""
151+
if instance_config is None:
152+
instance_config = config["providers"]["gcp"]["instances"]["default"]
153+
154+
gcp_check_delete(instance_config)
155+
gcp_check_stop(instance_config)
156+
gcp_deployment(instance_config["scaling"]["min_scaling"], instance_config)
157+
ip_ready = gcp_check_alive(instance_config)
109158
return ip_ready

‎cloudproxy/providers/settings.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,14 +148,15 @@
148148
config["providers"]["gcp"]["instances"]["default"]["secrets"]["service_account_key"] = os.environ.get(
149149
"GCP_SERVICE_ACCOUNT_KEY"
150150
)
151+
config["providers"]["gcp"]["instances"]["default"]["secrets"]["sa_json"] = os.environ.get("GCP_SA_JSON")
151152
config["providers"]["gcp"]["instances"]["default"]["scaling"]["min_scaling"] = int(
152153
os.environ.get("GCP_MIN_SCALING", 2)
153154
)
154155
config["providers"]["gcp"]["instances"]["default"]["scaling"]["max_scaling"] = int(
155156
os.environ.get("GCP_MAX_SCALING", 2)
156157
)
157158
config["providers"]["gcp"]["instances"]["default"]["size"] = os.environ.get("GCP_SIZE", "f1-micro")
158-
config["providers"]["gcp"]["instances"]["default"]["zone"] = os.environ.get("GCP_REGION", "us-central1-a")
159+
config["providers"]["gcp"]["instances"]["default"]["zone"] = os.environ.get("GCP_ZONE", "us-central1-a")
159160
config["providers"]["gcp"]["instances"]["default"]["image_project"] = os.environ.get("GCP_IMAGE_PROJECT", "ubuntu-os-cloud")
160161
config["providers"]["gcp"]["instances"]["default"]["image_family"] = os.environ.get("GCP_IMAGE_FAMILY", "ubuntu-minimal-2004-lts")
161162
config["providers"]["gcp"]["instances"]["default"]["display_name"] = os.environ.get("GCP_DISPLAY_NAME", "GCP")

‎docs/gcp.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,9 @@ Now you have your credentials, you can use GCP as a proxy provider. Set up the e
2121
#### Required:
2222
``GCP_ENABLED`` - to enable GCP as a provider, set as True. Default value: False
2323

24-
``GCP_SA_JSON`` - the path to the service account JSON key file. For Docker, mount the file to the container and provide the path.
24+
``GCP_SA_JSON`` or `GCP_SERVICE_ACCOUNT_KEY` - the path to the service account JSON key file. For Docker, mount the file to the container and provide the path. If both `GCP_SA_JSON` and `GCP_SERVICE_ACCOUNT_KEY` are set, `GCP_SA_JSON` will override the other.
25+
26+
``GCP_SERVICE_ACCOUNT_KEY`` or (`GCP_SA_JSON`) - service acount JSON key content. If both `GCP_SA_JSON` and `GCP_SERVICE_ACCOUNT_KEY` are set, `GCP_SA_JSON` will override the other.
2527

2628
``GCP_ZONE`` - the GCP zone where the instances will be created. Default value: us-central1-a
2729

0 commit comments

Comments
 (0)