From 2bcf21ad41f89795d2c269ce88ab2d4d3bb2c9a9 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Tue, 18 Feb 2025 12:29:05 +1100 Subject: [PATCH 1/7] Use apache client in GCS repository --- gradle/verification-metadata.xml | 5 + modules/repository-gcs/build.gradle | 49 +- .../licenses/httpclient-LICENSE.txt | 558 ++++++++++++++++++ .../licenses/httpclient-NOTICE.txt | 6 + .../licenses/httpcore-LICENSE.txt | 178 ++++++ .../licenses/httpcore-NOTICE.txt | 5 + ...eCloudStorageBlobStoreRepositoryTests.java | 4 +- .../GoogleCloudStorageHttpStatsCollector.java | 32 +- .../gcs/GoogleCloudStoragePlugin.java | 17 +- .../gcs/GoogleCloudStorageService.java | 82 ++- ...CloudStorageBlobContainerRetriesTests.java | 2 +- .../gcs/GoogleCloudStorageServiceTests.java | 2 +- 12 files changed, 848 insertions(+), 92 deletions(-) create mode 100644 modules/repository-gcs/licenses/httpclient-LICENSE.txt create mode 100644 modules/repository-gcs/licenses/httpclient-NOTICE.txt create mode 100644 modules/repository-gcs/licenses/httpcore-LICENSE.txt create mode 100644 modules/repository-gcs/licenses/httpcore-NOTICE.txt diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index eb5776d0e96ef..a72cb95f9848c 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -699,6 +699,11 @@ + + + + + diff --git a/modules/repository-gcs/build.gradle b/modules/repository-gcs/build.gradle index d23a0f4a7e44d..9f830b8000401 100644 --- a/modules/repository-gcs/build.gradle +++ b/modules/repository-gcs/build.gradle @@ -52,9 +52,9 @@ dependencies { api 'io.opencensus:opencensus-api:0.31.1' api 'io.opencensus:opencensus-contrib-http-util:0.31.1' api 'com.google.apis:google-api-services-storage:v1-rev20220705-2.0.0' - - testImplementation "org.apache.httpcomponents:httpclient:${versions.httpclient}" - testImplementation "org.apache.httpcomponents:httpcore:${versions.httpcore}" + api 'com.google.http-client:google-http-client-apache-v2:1.42.3' + api "org.apache.httpcomponents:httpclient:${versions.httpclient}" + api "org.apache.httpcomponents:httpcore:${versions.httpcore}" testImplementation project(':test:fixtures:gcs-fixture') yamlRestTestImplementation project(':test:fixtures:gcs-fixture') @@ -103,7 +103,6 @@ tasks.named("thirdPartyAudit").configure { ) ignoreMissingClasses( - 'com.google.api.client.http.apache.v2.ApacheHttpTransport', 'com.google.appengine.api.datastore.Blob', 'com.google.appengine.api.datastore.DatastoreService', 'com.google.appengine.api.datastore.DatastoreServiceFactory', @@ -129,48 +128,6 @@ tasks.named("thirdPartyAudit").configure { 'org.apache.log.Logger', 'javax.jms.Message', - // optional apache http client dependencies - 'org.apache.http.ConnectionReuseStrategy', - 'org.apache.http.Header', - 'org.apache.http.HttpEntity', - 'org.apache.http.HttpEntityEnclosingRequest', - 'org.apache.http.HttpHost', - 'org.apache.http.HttpRequest', - 'org.apache.http.HttpResponse', - 'org.apache.http.HttpVersion', - 'org.apache.http.RequestLine', - 'org.apache.http.StatusLine', - 'org.apache.http.client.AuthenticationHandler', - 'org.apache.http.client.HttpClient', - 'org.apache.http.client.HttpRequestRetryHandler', - 'org.apache.http.client.RedirectHandler', - 'org.apache.http.client.RequestDirector', - 'org.apache.http.client.UserTokenHandler', - 'org.apache.http.client.methods.HttpEntityEnclosingRequestBase', - 'org.apache.http.client.methods.HttpRequestBase', - 'org.apache.http.config.Registry', - 'org.apache.http.config.RegistryBuilder', - 'org.apache.http.conn.ClientConnectionManager', - 'org.apache.http.conn.ConnectionKeepAliveStrategy', - 'org.apache.http.conn.params.ConnManagerParams', - 'org.apache.http.conn.params.ConnRouteParams', - 'org.apache.http.conn.routing.HttpRoutePlanner', - 'org.apache.http.conn.scheme.PlainSocketFactory', - 'org.apache.http.conn.scheme.SchemeRegistry', - 'org.apache.http.conn.socket.PlainConnectionSocketFactory', - 'org.apache.http.conn.ssl.SSLSocketFactory', - 'org.apache.http.conn.ssl.X509HostnameVerifier', - 'org.apache.http.entity.AbstractHttpEntity', - 'org.apache.http.impl.client.DefaultHttpClient', - 'org.apache.http.impl.client.HttpClientBuilder', - 'org.apache.http.impl.conn.PoolingHttpClientConnectionManager', - 'org.apache.http.params.HttpConnectionParams', - 'org.apache.http.params.HttpParams', - 'org.apache.http.params.HttpProtocolParams', - 'org.apache.http.protocol.HttpContext', - 'org.apache.http.protocol.HttpProcessor', - 'org.apache.http.protocol.HttpRequestExecutor', - // commons-logging provided dependencies 'javax.servlet.ServletContextEvent', 'javax.servlet.ServletContextListener' diff --git a/modules/repository-gcs/licenses/httpclient-LICENSE.txt b/modules/repository-gcs/licenses/httpclient-LICENSE.txt new file mode 100644 index 0000000000000..32f01eda18fe9 --- /dev/null +++ b/modules/repository-gcs/licenses/httpclient-LICENSE.txt @@ -0,0 +1,558 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + +========================================================================= + +This project includes Public Suffix List copied from + +licensed under the terms of the Mozilla Public License, v. 2.0 + +Full license text: + +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/modules/repository-gcs/licenses/httpclient-NOTICE.txt b/modules/repository-gcs/licenses/httpclient-NOTICE.txt new file mode 100644 index 0000000000000..91e5c40c4c6d3 --- /dev/null +++ b/modules/repository-gcs/licenses/httpclient-NOTICE.txt @@ -0,0 +1,6 @@ +Apache HttpComponents Client +Copyright 1999-2016 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + diff --git a/modules/repository-gcs/licenses/httpcore-LICENSE.txt b/modules/repository-gcs/licenses/httpcore-LICENSE.txt new file mode 100644 index 0000000000000..e454a52586f29 --- /dev/null +++ b/modules/repository-gcs/licenses/httpcore-LICENSE.txt @@ -0,0 +1,178 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + diff --git a/modules/repository-gcs/licenses/httpcore-NOTICE.txt b/modules/repository-gcs/licenses/httpcore-NOTICE.txt new file mode 100644 index 0000000000000..013448d3e9561 --- /dev/null +++ b/modules/repository-gcs/licenses/httpcore-NOTICE.txt @@ -0,0 +1,5 @@ +Apache HttpComponents Core +Copyright 2005-2016 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java index 1adc380216529..4ea9f6b4f7f29 100644 --- a/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java +++ b/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java @@ -212,8 +212,8 @@ public TestGoogleCloudStoragePlugin(Settings settings) { } @Override - protected GoogleCloudStorageService createStorageService() { - return new GoogleCloudStorageService() { + protected GoogleCloudStorageService createStorageService(Settings settings) { + return new GoogleCloudStorageService(settings) { @Override StorageOptions createStorageOptions( final GoogleCloudStorageClientSettings gcsClientSettings, diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageHttpStatsCollector.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageHttpStatsCollector.java index e7e21360de545..23cd8a9b1f893 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageHttpStatsCollector.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageHttpStatsCollector.java @@ -9,10 +9,12 @@ package org.elasticsearch.repositories.gcs; -import com.google.api.client.http.GenericUrl; -import com.google.api.client.http.HttpRequest; -import com.google.api.client.http.HttpResponse; -import com.google.api.client.http.HttpResponseInterceptor; +import org.apache.http.HttpRequest; +import org.apache.http.HttpResponse; +import org.apache.http.HttpResponseInterceptor; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.protocol.HttpContext; +import org.elasticsearch.rest.RestStatus; import java.util.List; import java.util.Locale; @@ -53,11 +55,13 @@ final class GoogleCloudStorageHttpStatsCollector implements HttpResponseIntercep } @Override - public void interceptResponse(final HttpResponse response) { + public void process(HttpResponse response, HttpContext context) { // TODO keep track of unsuccessful requests in different entries - if (response.isSuccessStatusCode() == false) return; - - final HttpRequest request = response.getRequest(); + if (RestStatus.isSuccessful(response.getStatusLine().getStatusCode()) == false) { + return; + } + final HttpClientContext clientContext = HttpClientContext.adapt(context); + final HttpRequest request = clientContext.getRequest(); for (HttpRequestTracker tracker : trackers) { if (tracker.track(request, gcsOperationStats)) { return; @@ -109,11 +113,17 @@ private boolean track(final HttpRequest httpRequest, final GoogleCloudStorageOpe } private boolean matchesCriteria(final HttpRequest httpRequest) { - return method.equalsIgnoreCase(httpRequest.getRequestMethod()) && pathMatches(httpRequest.getUrl()); + return method.equalsIgnoreCase(httpRequest.getRequestLine().getMethod()) + && pathMatches(stripRequestParameters(httpRequest.getRequestLine().getUri())); + } + + private boolean pathMatches(final String url) { + return pathPattern.matcher(url).matches(); } - private boolean pathMatches(final GenericUrl url) { - return pathPattern.matcher(url.getRawPath()).matches(); + private String stripRequestParameters(String uri) { + final int queryIndex = uri.indexOf('?'); + return queryIndex == -1 ? uri : uri.substring(0, queryIndex); } } } diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java index 14281b7288067..4fad7c1662ddb 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.core.IOUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.plugins.Plugin; @@ -22,6 +23,7 @@ import org.elasticsearch.repositories.Repository; import org.elasticsearch.xcontent.NamedXContentRegistry; +import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -34,14 +36,14 @@ public class GoogleCloudStoragePlugin extends Plugin implements RepositoryPlugin @SuppressWarnings("this-escape") public GoogleCloudStoragePlugin(final Settings settings) { - this.storageService = createStorageService(); + this.storageService = createStorageService(settings); // eagerly load client settings so that secure settings are readable (not closed) reload(settings); } // overridable for tests - protected GoogleCloudStorageService createStorageService() { - return new GoogleCloudStorageService(); + protected GoogleCloudStorageService createStorageService(Settings settings) { + return new GoogleCloudStorageService(settings); } @Override @@ -78,7 +80,8 @@ public List> getSettings() { GoogleCloudStorageClientSettings.TOKEN_URI_SETTING, GoogleCloudStorageClientSettings.PROXY_TYPE_SETTING, GoogleCloudStorageClientSettings.PROXY_HOST_SETTING, - GoogleCloudStorageClientSettings.PROXY_PORT_SETTING + GoogleCloudStorageClientSettings.PROXY_PORT_SETTING, + GoogleCloudStorageService.MAX_OPEN_CONNECTIONS ); } @@ -92,4 +95,10 @@ public void reload(Settings settings) { final Map clientsSettings = GoogleCloudStorageClientSettings.load(settings); this.storageService.refreshAndClearCache(clientsSettings); } + + @Override + public void close() throws IOException { + IOUtils.closeWhileHandlingException(this.storageService); + super.close(); + } } diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java index 6a4eeeeabbb6f..1c029017a8a7b 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java @@ -10,9 +10,8 @@ package org.elasticsearch.repositories.gcs; import com.google.api.client.googleapis.GoogleUtils; -import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; -import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.http.apache.v2.ApacheHttpTransport; import com.google.api.client.util.SecurityUtils; import com.google.auth.oauth2.GoogleCredentials; import com.google.auth.oauth2.ServiceAccountCredentials; @@ -22,15 +21,23 @@ import com.google.cloud.storage.StorageOptions; import com.google.cloud.storage.StorageRetryStrategy; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.ssl.SSLContexts; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.Maps; +import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; import java.io.BufferedReader; +import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -40,15 +47,30 @@ import java.net.URL; import java.security.KeyStore; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import javax.net.ssl.SSLContext; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Collections.emptyMap; import static org.elasticsearch.core.Strings.format; -public class GoogleCloudStorageService { +public class GoogleCloudStorageService implements Closeable { private static final Logger logger = LogManager.getLogger(GoogleCloudStorageService.class); + private static final int DEFAULT_MAX_CONNECTIONS = 50; + + static final Setting MAX_OPEN_CONNECTIONS = Setting.intSetting( + "repository.gcs.http_client.max_open_connections", + DEFAULT_MAX_CONNECTIONS, + 1, + Setting.Property.NodeScope + ); + + private final AtomicReference connectionManager = new AtomicReference<>(); + private final int maxOpenConnections; + private volatile Map clientSettings = emptyMap(); /** @@ -58,6 +80,10 @@ public class GoogleCloudStorageService { */ private volatile Map clientCache = emptyMap(); + public GoogleCloudStorageService(Settings settings) { + maxOpenConnections = MAX_OPEN_CONNECTIONS.get(settings); + } + /** * Refreshes the client settings and clears the client cache. Subsequent calls to * {@code GoogleCloudStorageService#client} will return new clients constructed @@ -131,7 +157,6 @@ synchronized void closeRepositoryClient(String repositoryName) { private Storage createClient(GoogleCloudStorageClientSettings gcsClientSettings, GoogleCloudStorageOperationsStats stats) throws IOException { final HttpTransport httpTransport = SocketAccess.doPrivilegedIOException(() -> { - final NetHttpTransport.Builder builder = new NetHttpTransport.Builder(); // requires java.lang.RuntimePermission "setFactory" // Pin the TLS trust certificates. // We manually load the key store from jks instead of using GoogleUtils.getCertificateTrustStore() because that uses a .p12 @@ -140,36 +165,34 @@ private Storage createClient(GoogleCloudStorageClientSettings gcsClientSettings, try (InputStream keyStoreStream = GoogleUtils.class.getResourceAsStream("google.jks")) { SecurityUtils.loadKeyStore(certTrustStore, keyStoreStream, "notasecret"); } - builder.trustCertificates(certTrustStore); - Proxy proxy = gcsClientSettings.getProxy(); - if (proxy != null) { - builder.setProxy(proxy); - notifyProxyIsSet(proxy); - } - return builder.build(); - }); + final SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(certTrustStore, null).build(); + final GoogleCloudStorageHttpStatsCollector httpStatsCollector = new GoogleCloudStorageHttpStatsCollector(stats); + final HttpClientBuilder builder = HttpClients.custom() + .addInterceptorLast(httpStatsCollector) + .setSSLContext(sslContext) + .setConnectionManager(connectionManager.updateAndGet(existing -> { + if (existing == null) { + final PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(); + connectionManager.setMaxTotal(maxOpenConnections); + connectionManager.setDefaultMaxPerRoute(maxOpenConnections); + return connectionManager; + } + return existing; + })) + .setConnectionManagerShared(true) + // ApacheHttpTransport indicates to disable redirect and retries + .disableRedirectHandling() + .disableAutomaticRetries(); - final GoogleCloudStorageHttpStatsCollector httpStatsCollector = new GoogleCloudStorageHttpStatsCollector(stats); + return new ApacheHttpTransport(builder.build()); + }); final HttpTransportOptions httpTransportOptions = new HttpTransportOptions( HttpTransportOptions.newBuilder() .setConnectTimeout(toTimeout(gcsClientSettings.getConnectTimeout())) .setReadTimeout(toTimeout(gcsClientSettings.getReadTimeout())) .setHttpTransportFactory(() -> httpTransport) - ) { - - @Override - public HttpRequestInitializer getHttpRequestInitializer(ServiceOptions serviceOptions) { - HttpRequestInitializer requestInitializer = super.getHttpRequestInitializer(serviceOptions); - - return (httpRequest) -> { - if (requestInitializer != null) requestInitializer.initialize(httpRequest); - - httpRequest.setResponseInterceptor(httpStatsCollector); - }; - } - }; - + ); final StorageOptions storageOptions = createStorageOptions(gcsClientSettings, httpTransportOptions); return storageOptions.getService(); } @@ -280,4 +303,9 @@ static Integer toTimeout(final TimeValue timeout) { // used for unit testing void notifyProxyIsSet(Proxy proxy) {} + + @Override + public void close() { + IOUtils.closeWhileHandlingException(connectionManager.get()); + } } diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java index 96db51a060f4c..0198f443cd693 100644 --- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java +++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java @@ -129,7 +129,7 @@ protected BlobContainer createBlobContainer( secureSettings.setFile(CREDENTIALS_FILE_SETTING.getConcreteSettingForNamespace(client).getKey(), createServiceAccount(random())); clientSettings.setSecureSettings(secureSettings); - final GoogleCloudStorageService service = new GoogleCloudStorageService() { + final GoogleCloudStorageService service = new GoogleCloudStorageService(Settings.EMPTY) { @Override StorageOptions createStorageOptions( final GoogleCloudStorageClientSettings gcsClientSettings, diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java index c6f6b0315f595..edb39cb3f26c2 100644 --- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java +++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java @@ -72,7 +72,7 @@ public void testClientInitializer() throws Exception { .put(GoogleCloudStorageClientSettings.PROXY_PORT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), 8080) .build(); SetOnce proxy = new SetOnce<>(); - final GoogleCloudStorageService service = new GoogleCloudStorageService() { + final GoogleCloudStorageService service = new GoogleCloudStorageService(Settings.EMPTY) { @Override void notifyProxyIsSet(Proxy p) { proxy.set(p); From 7958a694aec35b49fa89fb9968ab0b481f691d74 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Tue, 18 Feb 2025 12:54:30 +1100 Subject: [PATCH 2/7] Assert we have a reference to the request --- .../repositories/gcs/GoogleCloudStorageHttpStatsCollector.java | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageHttpStatsCollector.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageHttpStatsCollector.java index 23cd8a9b1f893..74ecdc1e14a76 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageHttpStatsCollector.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageHttpStatsCollector.java @@ -62,6 +62,7 @@ public void process(HttpResponse response, HttpContext context) { } final HttpClientContext clientContext = HttpClientContext.adapt(context); final HttpRequest request = clientContext.getRequest(); + assert request != null; for (HttpRequestTracker tracker : trackers) { if (tracker.track(request, gcsOperationStats)) { return; From cfeeb85388db384adb5a18594ef11e34a3d0e786 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Tue, 18 Feb 2025 13:07:31 +1100 Subject: [PATCH 3/7] Change dependency scope --- modules/repository-gcs/build.gradle | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/repository-gcs/build.gradle b/modules/repository-gcs/build.gradle index 9f830b8000401..1b3009d22aebf 100644 --- a/modules/repository-gcs/build.gradle +++ b/modules/repository-gcs/build.gradle @@ -52,9 +52,9 @@ dependencies { api 'io.opencensus:opencensus-api:0.31.1' api 'io.opencensus:opencensus-contrib-http-util:0.31.1' api 'com.google.apis:google-api-services-storage:v1-rev20220705-2.0.0' - api 'com.google.http-client:google-http-client-apache-v2:1.42.3' - api "org.apache.httpcomponents:httpclient:${versions.httpclient}" - api "org.apache.httpcomponents:httpcore:${versions.httpcore}" + implementation 'com.google.http-client:google-http-client-apache-v2:1.42.3' + implementation "org.apache.httpcomponents:httpclient:${versions.httpclient}" + implementation "org.apache.httpcomponents:httpcore:${versions.httpcore}" testImplementation project(':test:fixtures:gcs-fixture') yamlRestTestImplementation project(':test:fixtures:gcs-fixture') From c45459c01f0c0ae4fb8d82682dfaa0ec53f7b9d2 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Tue, 18 Feb 2025 13:11:08 +1100 Subject: [PATCH 4/7] Align notices --- modules/ingest-common/licenses/httpcore-NOTICE.txt | 7 ++----- modules/repository-gcs/licenses/httpcore-NOTICE.txt | 4 ++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/modules/ingest-common/licenses/httpcore-NOTICE.txt b/modules/ingest-common/licenses/httpcore-NOTICE.txt index c0be50a505ec1..e347932fbac1c 100644 --- a/modules/ingest-common/licenses/httpcore-NOTICE.txt +++ b/modules/ingest-common/licenses/httpcore-NOTICE.txt @@ -1,8 +1,5 @@ -Apache HttpComponents Core -Copyright 2005-2014 The Apache Software Foundation +Apache HttpCore +Copyright 2005-2022 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). - -This project contains annotations derived from JCIP-ANNOTATIONS -Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net diff --git a/modules/repository-gcs/licenses/httpcore-NOTICE.txt b/modules/repository-gcs/licenses/httpcore-NOTICE.txt index 013448d3e9561..e347932fbac1c 100644 --- a/modules/repository-gcs/licenses/httpcore-NOTICE.txt +++ b/modules/repository-gcs/licenses/httpcore-NOTICE.txt @@ -1,5 +1,5 @@ -Apache HttpComponents Core -Copyright 2005-2016 The Apache Software Foundation +Apache HttpCore +Copyright 2005-2022 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). From 3757768e55324442d2e0006986b07cca84be74e1 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Tue, 18 Feb 2025 13:22:38 +1100 Subject: [PATCH 5/7] Align notices --- client/rest/licenses/httpcore-NOTICE.txt | 4 ++-- modules/repository-s3/licenses/httpcore-NOTICE.txt | 7 ++----- modules/repository-url/licenses/httpcore-NOTICE.txt | 7 ++----- .../discovery-azure-classic/licenses/httpcore-NOTICE.txt | 7 ++----- plugins/discovery-ec2/licenses/httpcore-NOTICE.txt | 7 ++----- 5 files changed, 10 insertions(+), 22 deletions(-) diff --git a/client/rest/licenses/httpcore-NOTICE.txt b/client/rest/licenses/httpcore-NOTICE.txt index 013448d3e9561..e347932fbac1c 100644 --- a/client/rest/licenses/httpcore-NOTICE.txt +++ b/client/rest/licenses/httpcore-NOTICE.txt @@ -1,5 +1,5 @@ -Apache HttpComponents Core -Copyright 2005-2016 The Apache Software Foundation +Apache HttpCore +Copyright 2005-2022 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/modules/repository-s3/licenses/httpcore-NOTICE.txt b/modules/repository-s3/licenses/httpcore-NOTICE.txt index c0be50a505ec1..e347932fbac1c 100644 --- a/modules/repository-s3/licenses/httpcore-NOTICE.txt +++ b/modules/repository-s3/licenses/httpcore-NOTICE.txt @@ -1,8 +1,5 @@ -Apache HttpComponents Core -Copyright 2005-2014 The Apache Software Foundation +Apache HttpCore +Copyright 2005-2022 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). - -This project contains annotations derived from JCIP-ANNOTATIONS -Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net diff --git a/modules/repository-url/licenses/httpcore-NOTICE.txt b/modules/repository-url/licenses/httpcore-NOTICE.txt index c0be50a505ec1..e347932fbac1c 100644 --- a/modules/repository-url/licenses/httpcore-NOTICE.txt +++ b/modules/repository-url/licenses/httpcore-NOTICE.txt @@ -1,8 +1,5 @@ -Apache HttpComponents Core -Copyright 2005-2014 The Apache Software Foundation +Apache HttpCore +Copyright 2005-2022 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). - -This project contains annotations derived from JCIP-ANNOTATIONS -Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net diff --git a/plugins/discovery-azure-classic/licenses/httpcore-NOTICE.txt b/plugins/discovery-azure-classic/licenses/httpcore-NOTICE.txt index c0be50a505ec1..e347932fbac1c 100644 --- a/plugins/discovery-azure-classic/licenses/httpcore-NOTICE.txt +++ b/plugins/discovery-azure-classic/licenses/httpcore-NOTICE.txt @@ -1,8 +1,5 @@ -Apache HttpComponents Core -Copyright 2005-2014 The Apache Software Foundation +Apache HttpCore +Copyright 2005-2022 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). - -This project contains annotations derived from JCIP-ANNOTATIONS -Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net diff --git a/plugins/discovery-ec2/licenses/httpcore-NOTICE.txt b/plugins/discovery-ec2/licenses/httpcore-NOTICE.txt index c0be50a505ec1..e347932fbac1c 100644 --- a/plugins/discovery-ec2/licenses/httpcore-NOTICE.txt +++ b/plugins/discovery-ec2/licenses/httpcore-NOTICE.txt @@ -1,8 +1,5 @@ -Apache HttpComponents Core -Copyright 2005-2014 The Apache Software Foundation +Apache HttpCore +Copyright 2005-2022 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). - -This project contains annotations derived from JCIP-ANNOTATIONS -Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net From 2e67a4217fd1396eb3c4479902a44f5550626482 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Tue, 18 Feb 2025 14:12:23 +1100 Subject: [PATCH 6/7] Fix exposed settings test --- .../repositories/gcs/GoogleCloudStoragePluginTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePluginTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePluginTests.java index bd34018dd7821..8f90cbefc0fca 100644 --- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePluginTests.java +++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePluginTests.java @@ -32,7 +32,8 @@ public void testExposedSettings() { "gcs.client.*.token_uri", "gcs.client.*.proxy.type", "gcs.client.*.proxy.host", - "gcs.client.*.proxy.port" + "gcs.client.*.proxy.port", + "repository.gcs.http_client.max_open_connections" ), settings.stream().map(Setting::getKey).toList() ); From 686e6f4d4652317abbf466a1fecb9d01f670a6e4 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Tue, 18 Feb 2025 15:39:44 +1100 Subject: [PATCH 7/7] Minor tweaks --- .../repositories/gcs/GoogleCloudStorageService.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java index 1c029017a8a7b..bb23e2f08a336 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java @@ -179,6 +179,7 @@ private Storage createClient(GoogleCloudStorageClientSettings gcsClientSettings, } return existing; })) + .disableCookieManagement() .setConnectionManagerShared(true) // ApacheHttpTransport indicates to disable redirect and retries .disableRedirectHandling() @@ -187,12 +188,11 @@ private Storage createClient(GoogleCloudStorageClientSettings gcsClientSettings, return new ApacheHttpTransport(builder.build()); }); - final HttpTransportOptions httpTransportOptions = new HttpTransportOptions( - HttpTransportOptions.newBuilder() - .setConnectTimeout(toTimeout(gcsClientSettings.getConnectTimeout())) - .setReadTimeout(toTimeout(gcsClientSettings.getReadTimeout())) - .setHttpTransportFactory(() -> httpTransport) - ); + final HttpTransportOptions httpTransportOptions = HttpTransportOptions.newBuilder() + .setConnectTimeout(toTimeout(gcsClientSettings.getConnectTimeout())) + .setReadTimeout(toTimeout(gcsClientSettings.getReadTimeout())) + .setHttpTransportFactory(() -> httpTransport) + .build(); final StorageOptions storageOptions = createStorageOptions(gcsClientSettings, httpTransportOptions); return storageOptions.getService(); }