Skip to content

Commit ad163af

Browse files
STOYAN KIROVSTOYAN KIROV
STOYAN KIROV
authored and
STOYAN KIROV
committed
Refactor
1 parent f3aa571 commit ad163af

File tree

1 file changed

+62
-62
lines changed

1 file changed

+62
-62
lines changed

‎main.go‎

Lines changed: 62 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -21,18 +21,20 @@ type HistoryItem struct {
2121

2222
func main() {
2323
url := flag.String("url", "", "URL pattern to collect responses for")
24-
dateFrom := flag.String("from", "", "Date on which to start collecing responses. Inclusive. Format: yyyyMMddhhmmss. Defaults to first ever record.")
25-
dateTo := flag.String("to", "", "Date on which to end collecing responses. Inclusive. Format: yyyyMMddhhmmss. Defaults to last ever record.")
24+
dateFrom := flag.String("from", "", "Date on which to start collecing responses. Inclusive. Format: yyyyMMddhhmmss. Defaults to first ever record")
25+
dateTo := flag.String("to", "", "Date on which to end collecing responses. Inclusive. Format: yyyyMMddhhmmss. Defaults to last ever record")
2626
limit := flag.Int("limit", 0, "Limit the results")
2727
filter := flag.String("filter", "", "Filter your search, using the wayback cdx filters (find more here: https://github.com/internetarchive/wayback/tree/master/wayback-cdx-server#filtering)")
2828
collapse := flag.String("collapse", "", "A form of filtering, with which you can collaps adjasent fields(find more here: https://github.com/internetarchive/wayback/tree/master/wayback-cdx-server#collapsing)")
2929

30-
requestsPerSecond := flag.Int("req-per-sec", 0, "Requests per second. 0 means no one request at a time")
31-
estimateTime := flag.Bool("time", false, "Show how much time it would take to make all requests for the current query")
30+
requestsPerSecond := flag.Int("req-per-sec", 1, "Requests per second. Defaults to 1")
31+
estimateTime := flag.Bool("time", false, "Show how much time it would take to make all requests for the current query and exit (without response time take into account)")
3232
printUrls := flag.Bool("print-urls", false, "Print to stdout only a list of historic URLs, which you can request later")
3333
unique := flag.Bool("unique", false, "Print to stdout only unique reponses")
34-
output := flag.String("output", "", "Path to a folder where the tool will safe all unique responses in uniquely named files per response")
35-
logFile := flag.String("log-file", "", "Log every wayback history request url, but not the response")
34+
outputFolder := flag.String("output-folder", "", "Path to a folder where the tool will safe all unique responses, in uniquely named files")
35+
logSuccessFile := flag.String("log-success-file", "", "Path to log file. Log successful request urls only")
36+
logFailFile := flag.String("log-fail-file", "", "Path to log file. Log failed requests only")
37+
verbose := flag.Bool("verbose", false, "Show more detailed output")
3638

3739
flag.Parse()
3840

@@ -41,8 +43,8 @@ func main() {
4143
}
4244

4345
if (*printUrls && *unique) ||
44-
(*printUrls && *output != "") ||
45-
(*unique && *output != "") {
46+
(*printUrls && *outputFolder != "") ||
47+
(*unique && *outputFolder != "") {
4648
log.Fatal("you can only set one of the following arguments: print-urls, unique, output")
4749
}
4850

@@ -64,101 +66,88 @@ func main() {
6466
requestUrl += fmt.Sprintf("&filter=%v", *filter)
6567
}
6668

67-
addToLog("Main request url: "+requestUrl, *logFile)
69+
printVerbose(fmt.Sprintf("[*] Constructed main request url: %v", requestUrl), *verbose)
6870

6971
historyItems := getHistoryItems(requestUrl)
72+
requestsCount := len(historyItems)
73+
duration, err := time.ParseDuration(fmt.Sprintf("%vs", requestsCount/(*requestsPerSecond)))
74+
if err != nil {
75+
log.Fatalf("error parsing duration: %v", err)
76+
}
77+
78+
printVerbose(fmt.Sprintf("[*] Making %v requests for approx. %v", requestsCount, duration), *verbose)
7079

7180
if *estimateTime {
72-
requestsCount := len(historyItems)
73-
duration, err := time.ParseDuration(fmt.Sprintf("%vs", requestsCount/(*requestsPerSecond)))
74-
if err != nil {
75-
log.Fatalf("error parsing duration: %v", err)
81+
if *unique || *outputFolder != "" {
82+
fmt.Println("[!] With the unique or output options enabled, the request count(an therefore the time) will be much lower than what's show below")
7683
}
7784
fmt.Printf("All %v requests will be made in %v", requestsCount, duration)
85+
return
7886
}
7987

80-
var allHistoryUrls []string
8188
var wg sync.WaitGroup
82-
historicResponses := make([][]byte, len(historyItems))
89+
uniqueResponses := make(map[[20]byte][]byte)
8390
for i, hi := range historyItems {
8491
historyUrl := fmt.Sprintf("https://web.archive.org/web/%vif_/%v", hi.Timestamp, *url)
8592

8693
if *printUrls {
87-
allHistoryUrls = append(allHistoryUrls, historyUrl)
94+
fmt.Println(historyUrl)
8895
continue
8996
}
9097

9198
if i%*requestsPerSecond == 0 {
99+
printVerbose(fmt.Sprintf("[*] Throttoling for a second at request number %v", i), *verbose)
92100
time.Sleep(1 * time.Second)
93101
}
94102

95103
wg.Add(1)
96104
go func() {
97-
addToLog("Hisotry item request url: "+historyUrl, *logFile)
98-
99105
response, err := get(historyUrl)
100106
if err != nil {
101107
fmt.Printf("error making history item request: %v", err)
102-
addToLog(fmt.Sprintf("ERROR fetching %v: %v", historyUrl, err), *logFile)
108+
appendToFile(historyUrl, *logFailFile)
109+
return
110+
} else {
111+
appendToFile(historyUrl, *logSuccessFile)
103112
}
104113

105-
if !*printUrls && !*unique && *output == "" {
114+
hashedResponse := sha1.Sum(response)
115+
116+
if !*printUrls && !*unique && *outputFolder == "" {
106117
fmt.Println(string(response))
107118
}
108119

109-
historicResponses = append(historicResponses, response)
120+
if *unique {
121+
uniqueResponse := string(uniqueResponses[hashedResponse])
122+
if uniqueResponse == "" {
123+
printVerbose("[*] Found new unique response", *verbose)
124+
uniqueResponses[hashedResponse] = response
125+
fmt.Println(string(response))
126+
}
127+
}
128+
129+
if *outputFolder != "" && !fileExists(fmt.Sprintf("%v/%x", *outputFolder, hashedResponse)) {
130+
printVerbose(fmt.Sprintf("[*] Writing new unique response to file with name %x", hashedResponse), *verbose)
131+
os.MkdirAll(*outputFolder, os.ModePerm)
132+
appendToFile(string(response), fmt.Sprintf("%v/%x", *outputFolder, hashedResponse))
133+
}
110134

111135
wg.Done()
112136
}()
113137
}
114138
wg.Wait()
115-
116-
uniqueResponses := make(map[[20]byte][]byte)
117-
for _, res := range historicResponses {
118-
if *output != "" || *unique {
119-
uniqueResponses[sha1.Sum(res)] = res
120-
}
121-
122-
if !*unique && !*printUrls && *output == "" {
123-
fmt.Println(string(res))
124-
}
125-
}
126-
127-
if *unique {
128-
for k, _ := range uniqueResponses {
129-
fmt.Println(string(uniqueResponses[k]))
130-
}
131-
return
132-
}
133-
134-
if *printUrls {
135-
for _, au := range allHistoryUrls {
136-
fmt.Println(au)
137-
}
138-
return
139-
}
140-
141-
if *output != "" {
142-
os.MkdirAll(*output, 0700)
143-
for k, _ := range uniqueResponses {
144-
err := ioutil.WriteFile(fmt.Sprintf("%v/%x", *output, k), uniqueResponses[k], 0644)
145-
if err != nil {
146-
log.Fatalf("error writing to file: %v", err)
147-
}
148-
}
149-
}
150139
}
151140

152-
func addToLog(logRow string, logFile string) {
153-
if logFile != "" {
154-
f, err := os.OpenFile(logFile,
141+
func appendToFile(data string, filePath string) {
142+
if filePath != "" {
143+
f, err := os.OpenFile(filePath,
155144
os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
156145
if err != nil {
157-
log.Fatalf("error writing to log file: %v", err)
146+
log.Fatalf("error writing to file: %v", err)
158147
}
159148
defer f.Close()
160-
if _, err := f.WriteString(logRow + "\n"); err != nil {
161-
log.Fatalf("error writing to log file: %v", err)
149+
if _, err := f.WriteString(data + "\n"); err != nil {
150+
log.Fatalf("error writing to file: %v", err)
162151
}
163152
}
164153
}
@@ -204,3 +193,14 @@ func getHistoryItems(requestUrl string) []HistoryItem {
204193
}
205194
return timestamps
206195
}
196+
197+
func fileExists(name string) bool {
198+
_, err := os.Stat(name)
199+
return !os.IsNotExist(err)
200+
}
201+
202+
func printVerbose(msg string, verbose bool) {
203+
if verbose {
204+
fmt.Println(msg)
205+
}
206+
}

0 commit comments

Comments
 (0)