Skip to content

Commit 9d5754f

Browse files
web-flowweb-flow
committed
Stability improvements
1 parent 3f1e643 commit 9d5754f

File tree

2 files changed

+55
-65
lines changed

2 files changed

+55
-65
lines changed

‎is.go‎

Lines changed: 26 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ type IS struct {
3232

3333
submitid string
3434

35-
endpoint *url.URL
3635
httpClient *http.Client
3736
torClient *http.Client
3837
}
@@ -65,15 +64,18 @@ func init() {
6564
func (wbrc *Archiver) Wayback(links []string) (map[string]string, error) {
6665
collects, results := make(map[string]string), make(map[string]string)
6766
for _, link := range links {
68-
if !helper.IsURL(link) {
69-
logger.Info(link + " is invalid url.")
70-
continue
67+
if helper.IsURL(link) {
68+
collects[link] = link
7169
}
72-
collects[link] = link
70+
}
71+
if len(collects) == 0 {
72+
return results, fmt.Errorf("Not found")
7373
}
7474

75-
done := make(chan bool, 1)
76-
torClient, err := newTorClient(done)
75+
ctx, cancel := context.WithCancel(context.Background())
76+
defer cancel()
77+
torClient, t, err := newTorClient(ctx)
78+
defer closeTor(t)
7779
if err != nil {
7880
logger.Error("%v", err)
7981
}
@@ -102,11 +104,6 @@ func (wbrc *Archiver) Wayback(links []string) (map[string]string, error) {
102104
}
103105
wg.Wait()
104106

105-
// Close tor connection
106-
defer func() {
107-
done <- true
108-
}()
109-
110107
if len(results) == 0 {
111108
return results, fmt.Errorf("No results")
112109
}
@@ -123,11 +120,13 @@ func (wbrc *Archiver) Playback(links []string) (map[string]string, error) {
123120
}
124121
}
125122
if len(collects) == 0 {
126-
return results, fmt.Errorf("No found URL")
123+
return results, fmt.Errorf("Not found")
127124
}
128125

129-
done := make(chan bool, 1)
130-
torClient, err := newTorClient(done)
126+
ctx, cancel := context.WithCancel(context.Background())
127+
defer cancel()
128+
torClient, t, err := newTorClient(ctx)
129+
defer closeTor(t)
131130
if err != nil {
132131
logger.Error("%v", err)
133132
}
@@ -156,19 +155,14 @@ func (wbrc *Archiver) Playback(links []string) (map[string]string, error) {
156155
}
157156
wg.Wait()
158157

159-
// Close tor connection
160-
defer func() {
161-
done <- true
162-
}()
163-
164158
if len(results) == 0 {
165159
return results, fmt.Errorf("No results")
166160
}
167161

168162
return results, nil
169163
}
170164
func (is *IS) archive(uri string, ch chan<- string) {
171-
_, err := is.getValidDomain()
165+
endpoint, err := is.getValidDomain()
172166
if err != nil {
173167
ch <- fmt.Sprint("archive.today is unavailable.")
174168
return
@@ -182,14 +176,14 @@ func (is *IS) archive(uri string, ch chan<- string) {
182176
"anyway": {anyway},
183177
"url": {uri},
184178
}
185-
domain := is.endpoint.String()
179+
domain := endpoint.String()
186180
req, err := http.NewRequest("POST", domain+"/submit/", strings.NewReader(data.Encode()))
187181
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
188182
req.Header.Add("Content-Length", strconv.Itoa(len(data.Encode())))
189183
req.Header.Add("User-Agent", userAgent)
190184
req.Header.Add("Referer", domain)
191185
req.Header.Add("Origin", domain)
192-
req.Header.Add("Host", is.endpoint.Hostname())
186+
req.Header.Add("Host", endpoint.Hostname())
193187
req.Header.Add("Cookie", is.getCookie())
194188
resp, err := is.httpClient.Do(req)
195189
if err != nil {
@@ -286,6 +280,7 @@ func (is *IS) getSubmitID(url string) (string, error) {
286280
}
287281

288282
func (is *IS) getValidDomain() (*url.URL, error) {
283+
var endpoint *url.URL
289284
// get valid domain and submitid
290285
r := func(domains []string) {
291286
for _, domain := range domains {
@@ -294,8 +289,8 @@ func (is *IS) getValidDomain() (*url.URL, error) {
294289
if err != nil {
295290
continue
296291
}
297-
is.endpoint, _ = url.Parse(h)
298292
is.submitid = id
293+
endpoint, _ = url.Parse(h)
299294
break
300295
}
301296
}
@@ -307,28 +302,28 @@ func (is *IS) getValidDomain() (*url.URL, error) {
307302
r([]string{onion})
308303
}
309304

310-
if is.endpoint == nil || is.submitid == "" {
305+
if endpoint == nil || is.submitid == "" {
311306
r(domains)
312-
if is.endpoint == nil || is.submitid == "" {
307+
if endpoint == nil || is.submitid == "" {
313308
return nil, fmt.Errorf("archive.today is unavailable.")
314309
}
315310
}
316311

317-
return is.endpoint, nil
312+
return endpoint, nil
318313
}
319314

320315
func (is *IS) search(uri string, ch chan<- string) {
321-
_, err := is.getValidDomain()
316+
endpoint, err := is.getValidDomain()
322317
if err != nil {
323318
ch <- fmt.Sprint("archive.today is unavailable.")
324319
return
325320
}
326321

327-
domain := is.endpoint.String()
322+
domain := endpoint.String()
328323
req, err := http.NewRequest("GET", fmt.Sprintf("%s/%s", domain, uri), nil)
329324
req.Header.Add("User-Agent", userAgent)
330325
req.Header.Add("Referer", domain)
331-
req.Header.Add("Host", is.endpoint.Hostname())
326+
req.Header.Add("Host", endpoint.Hostname())
332327
resp, err := is.httpClient.Do(req)
333328
if err != nil {
334329
ch <- fmt.Sprint(err)
@@ -344,7 +339,7 @@ func (is *IS) search(uri string, ch chan<- string) {
344339

345340
target, exists := doc.Find("#row0 > .TEXT-BLOCK > a").Attr("href")
346341
if !exists {
347-
ch <- "No found"
342+
ch <- "Not found"
348343
return
349344
}
350345

‎tor.go‎

Lines changed: 29 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,14 @@ import (
1616
"golang.org/x/net/proxy"
1717
)
1818

19-
func newTorClient(done <-chan bool) (*http.Client, error) {
19+
func newTorClient(ctx context.Context) (client *http.Client, t *tor.Tor, err error) {
2020
var dialer proxy.ContextDialer
21-
if useProxy() {
21+
addr, isUseProxy := useProxy()
22+
if isUseProxy {
2223
// Create a socks5 dialer
23-
pxy, err := proxy.SOCKS5("tcp", "127.0.0.1:9050", nil, proxy.Direct)
24+
pxy, err := proxy.SOCKS5("tcp", addr, nil, proxy.Direct)
2425
if err != nil {
25-
return nil, fmt.Errorf("Can't connect to the proxy: %w", err)
26+
return nil, t, fmt.Errorf("Can't connect to the proxy: %w", err)
2627
}
2728

2829
dialer = pxy.(interface {
@@ -31,44 +32,30 @@ func newTorClient(done <-chan bool) (*http.Client, error) {
3132
} else {
3233
// Lookup tor executable file
3334
if _, err := exec.LookPath("tor"); err != nil {
34-
return nil, fmt.Errorf("%w", err)
35+
return nil, t, fmt.Errorf("%w", err)
3536
}
3637

3738
// Start tor with default config
38-
startConf := &tor.StartConf{TempDataDirBase: os.TempDir()}
39-
t, err := tor.Start(nil, startConf)
39+
startConf := &tor.StartConf{TempDataDirBase: os.TempDir(), RetainTempDataDir: false, NoHush: false}
40+
t, err = tor.Start(nil, startConf)
4041
if err != nil {
41-
return nil, fmt.Errorf("Make connection failed: %w", err)
42+
return nil, t, fmt.Errorf("Make connection failed: %w", err)
4243
}
4344
// defer t.Close()
45+
t.DeleteDataDirOnClose = true
46+
t.StopProcessOnClose = true
4447

4548
// Wait at most a minute to start network and get
46-
dialCtx, dialCancel := context.WithTimeout(context.Background(), time.Minute)
49+
dialCtx, dialCancel := context.WithTimeout(ctx, time.Minute)
4750
defer dialCancel()
51+
// t.ProcessCancelFunc = dialCancel
4852

4953
// Make connection
5054
dialer, err = t.Dialer(dialCtx, nil)
5155
if err != nil {
5256
t.Close()
53-
return nil, fmt.Errorf("Make connection failed: %w", err)
57+
return nil, t, fmt.Errorf("Make connection failed: %w", err)
5458
}
55-
56-
go func() {
57-
for {
58-
select {
59-
case <-done:
60-
logger.Debug("Closed tor client")
61-
t.Close()
62-
return
63-
case <-time.After(10 * time.Minute):
64-
logger.Debug("Closed tor client, timeout")
65-
t.Close()
66-
return
67-
default:
68-
logger.Debug("Waiting for close tor client")
69-
}
70-
}
71-
}()
7259
}
7360

7461
return &http.Client{
@@ -86,10 +73,17 @@ func newTorClient(done <-chan bool) (*http.Client, error) {
8673
InsecureSkipVerify: true,
8774
},
8875
},
89-
}, nil
76+
}, t, nil
77+
}
78+
79+
func closeTor(t *tor.Tor) error {
80+
if t != nil {
81+
t.Close()
82+
}
83+
return nil
9084
}
9185

92-
func useProxy() bool {
86+
func useProxy() (addr string, ok bool) {
9387
host := os.Getenv("TOR_HOST")
9488
port := os.Getenv("TOR_SOCKS_PORT")
9589
if host == "" {
@@ -99,16 +93,17 @@ func useProxy() bool {
9993
port = "9050"
10094
}
10195

102-
conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, port), time.Second)
96+
addr = net.JoinHostPort(host, port)
97+
conn, err := net.DialTimeout("tcp", addr, time.Second)
10398
if err != nil {
10499
logger.Debug("Try to connect tor proxy failed: %v", err)
105-
return false
100+
return addr, false
106101
}
107102
if conn != nil {
108103
conn.Close()
109-
logger.Debug("Connected: %v", net.JoinHostPort(host, port))
110-
return true
104+
logger.Debug("Connected: %v", addr)
105+
return addr, true
111106
}
112107

113-
return false
108+
return addr, false
114109
}

0 commit comments

Comments
 (0)