FishGlob_data/cleaning_codes/merge.R at main · fishglob/FishGlob_data · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
################################################################################
#### R code to merge all separate datasets
#### Coding: Juliano Palacios Abrantes & Aurore A. Maureaud, November 2023
################################################################################

# Load libraries
library(googledrive)
library(tidyverse)
library(ggplot2)
library(readr)
library(here)
library(readxl)
library(data.table)
library(R.utils)

# Load relevant functions
source(here("./functions/write_clean_data.r"))
source("./functions/read_clean_data.R")

#-------------------------------------------------------------------------------------------#
#### Read all processed survey files ####
#-------------------------------------------------------------------------------------------#
# Load cleaned surveys
surveys <- c("AI",
             "BITS",
             "EBS",
             "EVHOE",
             "FR-CGFS",
             "GMEX",
             "GOA",
             "GSL-N",
             "GSL-S",
             "HS",
             "IE-IGFS",
             "NEUS",
             "NIGFS",
             "NOR-BTS",
             "NS-IBTS",
             "PT-IBTS",
             "QCS",
             "ROCKALL",
             "SCS",
             "SP-ARSA",
             "SP-NORTH",
             "SP-PORC",
             "SWC-IBTS",
             "SOG",
             "SEUS",
             "WCANN",
             "WCHG",
             "WCTRI",
             "WCVI")

# create compiled dataset
fishglob <- read_clean_data(surveys)

# survey compiled survey product
# MLP 8 Dec 2025: had to delete outputs/Compiled_data/FishGlob_public_clean.csv.gz before running the next line (it is too large for github)
write_clean_data(data = fishglob, survey = "FishGlob_public",
                 overwrite = T, compiled = TRUE, gzip = FALSE)


#-------------------------------------------------------------------------------------------#
#### ADD STRANDARDIZATION FLAGS ####
#-------------------------------------------------------------------------------------------#

fishglob_std <- read_clean_data(surveys, std = TRUE)

# Just run this routine should be good for all
# MLP 8 Dec 2025: had to delete outputs/Compiled_data/FishGlob_public_std_clean.csv.gz before running the next line (too large for github)
write_clean_data(data = fishglob_std, survey = "FishGlob_public_std",
                 overwrite = T, compiled = TRUE, gzip = FALSE)


#-------------------------------------------------------------------------------------------#
#### METADATA PRODUCT ####
#-------------------------------------------------------------------------------------------#

fishglob_public_metadata <- fishglob %>%
  select(survey, latitude, longitude, year, survey_unit) %>%
  distinct()

# MLP 8 Dec 2025: had to delete outputs/Compiled_data/FishGlob_public_metadata_clean.csv.gz before running the next line
write_clean_data(data = fishglob_public_metadata,
                 survey = "FishGlob_public_metadata",
                 overwrite = TRUE, compiled = TRUE, gzip = FALSE)