Skip to content

Commit 3e8bf5a

Browse files
author
Andrea Spacca
committed
download zip package and load field yml from it
1 parent 0e15686 commit 3e8bf5a

File tree

3 files changed

+101
-76
lines changed

3 files changed

+101
-76
lines changed

‎pkg/genlib/fields/cache.go‎

Lines changed: 0 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,6 @@ import (
44
"context"
55
"golang.org/x/sync/semaphore"
66
"sync"
7-
8-
"github.com/elastic/go-ucfg/yaml"
97
)
108

119
const (
@@ -98,48 +96,3 @@ func (f *Cache) LoadFields(ctx context.Context, integration, stream, version str
9896

9997
return flds, nil
10098
}
101-
102-
func (f *Cache) LoadManifest(ctx context.Context, integration, stream, version string) (*Manifest, error) {
103-
var err error
104-
105-
t := tuple{
106-
integration: integration,
107-
stream: stream,
108-
version: version,
109-
}
110-
111-
f.mut.RLock()
112-
manifest, ok := f.manifest[t]
113-
f.mut.RUnlock()
114-
115-
if ok {
116-
return &manifest, nil
117-
}
118-
119-
// Pull the manifest file
120-
url, err := makeManifestURL(f.baseUrl, integration, stream, version)
121-
if err != nil {
122-
return nil, err
123-
}
124-
125-
data, err := getFromURL(ctx, url.String())
126-
if err != nil {
127-
return nil, err
128-
}
129-
130-
// deserialize
131-
cfg, err := yaml.NewConfig(data)
132-
if err != nil {
133-
return nil, err
134-
}
135-
err = cfg.Unpack(&manifest)
136-
if err != nil {
137-
return nil, err
138-
}
139-
140-
f.mut.Lock()
141-
f.manifest[t] = manifest
142-
f.mut.Unlock()
143-
144-
return &manifest, nil
145-
}

‎pkg/genlib/fields/load.go‎

Lines changed: 93 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,19 @@
11
package fields
22

33
import (
4+
"archive/zip"
45
"context"
6+
"crypto/sha256"
7+
"encoding/hex"
58
"encoding/json"
69
"errors"
710
"fmt"
11+
"github.com/elastic/elastic-integration-corpus-generator-tool/internal/settings"
12+
"io"
813
"io/ioutil"
914
"net/http"
1015
"net/url"
16+
"os"
1117
"path"
1218
"path/filepath"
1319
"strings"
@@ -26,12 +32,7 @@ const (
2632

2733
func LoadFields(ctx context.Context, baseURL, integration, dataStream, version string) (Fields, error) {
2834

29-
packageURL, err := makePackageURL(baseURL, integration, version)
30-
if err != nil {
31-
return nil, err
32-
}
33-
34-
fieldsContent, err := getFieldsFiles(ctx, packageURL, dataStream)
35+
fieldsContent, err := getFieldsFiles(ctx, baseURL, integration, dataStream, version)
3536
if err != nil {
3637
return nil, err
3738
}
@@ -61,42 +62,99 @@ func makePackageURL(baseURL, integration, version string) (*url.URL, error) {
6162
return u, nil
6263
}
6364

64-
func getFieldsFiles(ctx context.Context, packageURL *url.URL, dataStream string) ([]byte, error) {
65-
body, err := getFromURL(ctx, packageURL.String())
65+
func makeDownloadURL(baseURL, donwloadPath string) (*url.URL, error) {
66+
67+
u, err := url.Parse(baseURL)
6668
if err != nil {
6769
return nil, err
6870
}
6971

70-
var assetsPayload struct {
71-
Assets []string `json:"assets"`
72+
u.Path = path.Join(u.Path, donwloadPath)
73+
return u, nil
74+
}
75+
76+
func getFieldsFiles(ctx context.Context, baseURL, integration, dataStream, version string) ([]byte, error) {
77+
packageURL, err := makePackageURL(baseURL, integration, version)
78+
if err != nil {
79+
return nil, err
7280
}
7381

74-
if err = json.Unmarshal(body, &assetsPayload); err != nil {
82+
r, err := getFromURL(ctx, packageURL.String())
83+
if err != nil {
7584
return nil, err
7685
}
7786

78-
fieldsFilesURL := make([]string, 0)
79-
prefixFieldsPath := path.Join(packageURL.Path, dataStreamSlug, dataStream, fieldsSlug)
80-
for _, assetPath := range assetsPayload.Assets {
81-
if !strings.HasPrefix(assetPath, prefixFieldsPath) {
82-
continue
83-
}
87+
var downloadPayload struct {
88+
Download string `json:"download"`
89+
}
8490

85-
fieldsFilesURL = append(fieldsFilesURL, assetPath)
91+
body, err := ioutil.ReadAll(r)
92+
if err = json.Unmarshal(body, &downloadPayload); err != nil {
93+
return nil, err
8694
}
8795

96+
downloadURL, err := makeDownloadURL(baseURL, downloadPayload.Download)
97+
r, err = getFromURL(ctx, downloadURL.String())
98+
if err != nil {
99+
return nil, err
100+
}
101+
102+
h := sha256.New()
103+
h.Write([]byte(downloadURL.String()))
104+
prefix := hex.EncodeToString(h.Sum(nil))
105+
106+
packageTempDir, err := os.MkdirTemp(settings.CacheDir(), prefix)
107+
if err != nil {
108+
return nil, err
109+
}
110+
packageArchive := path.Join(packageTempDir, "package.zip")
111+
f, err := os.Create(packageArchive)
112+
if err != nil {
113+
return nil, err
114+
}
115+
116+
_, err = io.Copy(f, r)
117+
_ = f.Close()
118+
if err != nil {
119+
120+
return nil, err
121+
}
122+
123+
archive, err := zip.OpenReader(packageArchive)
124+
if err != nil {
125+
return nil, err
126+
}
127+
defer func() {
128+
_ = archive.Close()
129+
}()
130+
131+
prefixFieldsPath := path.Join(fmt.Sprintf("%s-%s", integration, version), dataStreamSlug, dataStream, fieldsSlug)
132+
88133
var fieldsContent string
89-
for _, fieldsFileURL := range fieldsFilesURL {
90-
packageURL.Path = fieldsFileURL
134+
for _, z := range archive.File {
135+
if z.FileInfo().IsDir() {
136+
continue
137+
}
138+
139+
if !strings.HasPrefix(z.Name, prefixFieldsPath) {
140+
continue
141+
}
91142

92-
body, err := getFromURL(ctx, packageURL.String())
143+
fieldsFileName := z.Name
144+
zr, err := z.Open()
93145
if err != nil {
94146
return nil, err
95147
}
96148

97-
key := strings.TrimSuffix(filepath.Base(fieldsFileURL), filepath.Ext(fieldsFileURL))
149+
fieldsFileContent, err := ioutil.ReadAll(zr)
150+
if err != nil {
151+
return nil, err
152+
}
153+
154+
_ = zr.Close()
155+
key := strings.TrimSuffix(filepath.Base(fieldsFileName), filepath.Ext(fieldsFileName))
98156
keyEntry := fmt.Sprintf("- key: %s\n fields:\n", key)
99-
for _, line := range strings.Split(string(body), "\n") {
157+
for _, line := range strings.Split(string(fieldsFileContent), "\n") {
100158
keyEntry += ` ` + line + "\n"
101159
}
102160

@@ -106,7 +164,7 @@ func getFieldsFiles(ctx context.Context, packageURL *url.URL, dataStream string)
106164
return []byte(fieldsContent), nil
107165
}
108166

109-
func getFromURL(ctx context.Context, srcURL string) ([]byte, error) {
167+
func getFromURL(ctx context.Context, srcURL string) (io.ReadCloser, error) {
110168

111169
req, err := http.NewRequestWithContext(ctx, "GET", srcURL, nil)
112170

@@ -116,18 +174,25 @@ func getFromURL(ctx context.Context, srcURL string) ([]byte, error) {
116174

117175
client := &http.Client{}
118176
resp, err := client.Do(req)
119-
120177
if err != nil {
178+
defer func(r io.ReadCloser) {
179+
if r != nil {
180+
_ = r.Close()
181+
}
182+
}(resp.Body)
121183
return nil, err
122184
}
123185

124-
defer resp.Body.Close()
125-
126186
if resp.StatusCode != http.StatusOK {
187+
defer func(r io.ReadCloser) {
188+
if r != nil {
189+
_ = r.Close()
190+
}
191+
}(resp.Body)
127192
return nil, ErrNotFound
128193
}
129194

130-
return ioutil.ReadAll(resp.Body)
195+
return resp.Body, nil
131196
}
132197

133198
func makeManifestURL(baseURL, integration, stream, version string) (*url.URL, error) {

‎pkg/genlib/fields/version.go‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"encoding/json"
66
"errors"
7+
"io/ioutil"
78
"net/url"
89
"path"
910
"strings"
@@ -17,7 +18,7 @@ func MapVersion(ctx context.Context, baseUrl, integration, kibanaVersion string)
1718
return "", err
1819
}
1920

20-
body, err := getFromURL(ctx, searchUrl.String())
21+
r, err := getFromURL(ctx, searchUrl.String())
2122
if err != nil {
2223
return "", err
2324
}
@@ -26,6 +27,12 @@ func MapVersion(ctx context.Context, baseUrl, integration, kibanaVersion string)
2627
Version string `json:"version"`
2728
}
2829

30+
body, err := ioutil.ReadAll(r)
31+
if err != nil {
32+
_ = r.Close()
33+
return "", err
34+
}
35+
2936
if err = json.Unmarshal(body, &payload); err != nil {
3037
return "", err
3138
}

0 commit comments

Comments
 (0)