diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ff880dc..941733f 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -25,7 +25,7 @@ jobs: version: - '1.10' - '1.11' - - 'nightly' + - '1.12' os: - ubuntu-latest - windows-latest diff --git a/.gitignore b/.gitignore index 24f9c89..95374f9 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,8 @@ JuliaLocalPreferences.toml main.jl settings.json .vscode/settings.json + +*.aux +*.log +summary_local.yaml +australia.yaml diff --git a/Project.toml b/Project.toml index 7557834..89207ae 100644 --- a/Project.toml +++ b/Project.toml @@ -17,14 +17,14 @@ ZipFile = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea" [compat] CSV = "0.10.15" -DataFrames = "1.7.0" -DataStructures = "0.18.22, 0.19" +DataFrames = "1.8.1" +DataStructures = "0.19" Downloads = "1.6.0" -JuMP = "1.28.0" -MPSGE = "0.6.1" +JuMP = "1.29" +MPSGE = "0.6.4" WiNDCContainer = "0.3.0" XLSX = "0.10.4" -YAML = "0.4.14" +YAML = "0.4" ZipFile = "0.10.1" julia = "1.10" diff --git a/README.md b/README.md index de4cc63..006cc4f 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ x -> sort(x, :diff) -M = national_mpsge(Y; year = 2023) +M = national_mpsge(Y) # add keyword year = 2023 to solve for a specific year MPSGE.solve!(M; cumulative_iteration_limit=0) # Solve at benchmark MPSGE.set_value!.(M[:Absorption_Tax], 0) # Set some taxes to 0 diff --git a/docs/assets/custom.css b/docs/assets/custom.css new file mode 100644 index 0000000..65dfc6d --- /dev/null +++ b/docs/assets/custom.css @@ -0,0 +1,3 @@ +img { + width: 75%; +} \ No newline at end of file diff --git a/docs/make.jl b/docs/make.jl index e568455..f97c96e 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,23 +1,38 @@ using Documenter, WiNDCNational, Literate +# Copy YAML Files to docs +cp(joinpath(@__DIR__, "..", "src", "united_states", "summary.yaml"), # Source + joinpath(@__DIR__, "src", "data", "united_states", "summary.yaml"); # `Dest + force = true) +cp(joinpath(@__DIR__, "..", "src", "united_states", "detailed.yaml"), # Source + joinpath(@__DIR__, "src", "data", "united_states", "detailed.yaml"); # `Dest + force = true) const _PAGES = [ "Introduction" => ["index.md"], "Data" => [ - "data_sources.md", - "data_build.md", - "sets.md" - ], - "National Module" => ["overview.md"], + "United States" => [ + "data/united_states/data_sources.md", + "data/united_states/data_build.md", + "data/united_states/sets.md", + "data/united_states/build_local.md" + ], + "Australia" => [ + "data/australia/australia_data.md", + "data/australia/load_data.md" + ], + ], + "Calibration" => ["calibration.md"], + "The National Model" => ["model/overview.md"], "API" => ["api.md"] ] literate_files = Dict( "sets" => ( - input = "src/sets.jl", - output = "src/" + input = "src/data/united_states/sets.jl", + output = "src/data/united_states/" ), ) @@ -35,9 +50,10 @@ end makedocs( sitename="WiNDCNational.jl", authors="Mitch Phillipson", - format = Documenter.HTML(), + format = Documenter.HTML(assets = ["assets/custom.css"]), modules = [WiNDCNational], - pages = _PAGES + pages = _PAGES, + ) diff --git a/docs/src/calibration.md b/docs/src/calibration.md new file mode 100644 index 0000000..2614c79 --- /dev/null +++ b/docs/src/calibration.md @@ -0,0 +1,5 @@ +# Overview + +The calibration routine uses a standard least-squares approach to fit model parameters to observed data. The `calibrate` function is inherited from `WiNDCContainer`, but requires specification of exogenous (fixed) variables and constraints to guide the calibration process. Variables are fixed by [`WiNDCNational.calibrate_fix_variables`](@ref) and the constraints are defined using [`WiNDCNational.calibrate_constraints`](@ref). + +The `calibrate` function returns a pair `(table, JuMP.Model)` where `table` is a WiNDCTable type and the `JuMP.Model` is the JuMP model used in the calibration. The JuMP model can be used to determine if the model calibrates and debug the constraints if it does not. diff --git a/docs/src/data/australia/australia_data.md b/docs/src/data/australia/australia_data.md new file mode 100644 index 0000000..663c797 --- /dev/null +++ b/docs/src/data/australia/australia_data.md @@ -0,0 +1,13 @@ +# [Data Sources](@id australia_data_sources) + +We use the Supply and Use tables provided by the Australian Bureau of Statistics (ABS). + +## Australian Bureau of Statistics - Supply and Use Tables + +Scroll down to the `Data Downloads` section of the +[ABS Website](https://www.abs.gov.au/statistics/economy/national-accounts/australian-national-accounts-supply-use-tables/latest-release) and download the three tables: +- `Supply Table` - The supply table +- `Use Table` - The use table +- `Primary Inputs Table` - Value Added + +This data is only available in `XLSB` format, which Julia is unable to open. You must first manually convert the file to `XLSX` format using Excel. \ No newline at end of file diff --git a/docs/src/data/australia/australia_local.yaml b/docs/src/data/australia/australia_local.yaml new file mode 100644 index 0000000..01d26f7 --- /dev/null +++ b/docs/src/data/australia/australia_local.yaml @@ -0,0 +1,282 @@ +metadata: + years: + "2021-22": 2022 + "2022-23": 2023 + "2023-24": 2024 + paths: + use: + path: /path/to/australia_use_summary.xlsx + supply: + path: /path/to/australia_supply_summary.xlsx + value_added: + path: /path/to/australia_value_added_summary.xlsx + transformation_keywords: + insurance_codes: + - "6301" + na_values: + - "n.p." +sets: + commodity: + description: "Commodities" + domain: "row" + values : "A8:A122" + descriptions : "B8:B122" + table: "supply" + sector: + description: "Sectors" + domain: "col" + values : "C6:BR6" + descriptions : "C7:BR7" + table: "supply" + import: + description: "Imports" + domain: "col" + values: "BT7" + descriptions: "BT7" + value_labels: + - import + table: "supply" + trade: + description: "Trade" + domain: "col" + values: "BW6:BZ6" + descriptions: "BW7:BZ7" + value_labels: + - retail + - wholesale + - electric + - gas + table: "supply" + transport: + description: "Transport" + domain: "col" + values: "BV7" + descriptions: "BV7" + value_labels: + - transport + table: "supply" + tax: + description: "Tax" + domain: "col" + values: "CB7" + descriptions: "CB7" + value_labels: + - tax + table: "supply" + + + personal_consumption: + description: "Personal consumption expenditures" + domain: "col" + values: "BT7" + descriptions: "BT7" + value_labels: + - pce + table: "use" + investment_final_demand: + description: "Investment portions of final demand" + domain: "col" + values: "BV7:BX7" + descriptions: "BV7:BX7" + value_labels: + - private + - public + - general + table: "use" + export: + description: "Exports" + domain: "col" + values: "BZ6" + descriptions: "BZ7" + value_labels: + - export + table: "use" + reexport: + description: "Re-exports" + domain: "col" + values: "CA7" + descriptions: "CA7" + value_labels: + - reexport + table: "use" + government_final_demand: + description: "Government portions of final demand" + domain: "col" + values: "BU7" + descriptions: "BU7" + value_labels: + - gce + table: "use" + + labor_demand: + description: "Compensation of employees" + domain: "row" + values: "A8" + descriptions: "A8" + value_labels: + - labor + table: "value_added" + capital_demand: + description: "Gross operating surplus" + domain: "row" + values: "A9" + descriptions: "A9" + value_labels: + - capital + table: "value_added" + output_tax: + description: "Output tax" + domain: "row" + values: "A11" + descriptions: "A11" + value_labels: + - output_tax + table: "value_added" + +parameters: + intermediate_supply: + param: "Intermediate_Supply" + description: "" + row: "commodity" + col: "sector" + table: "supply" + import: + param: "Import" + description: "" + row: "commodity" + col: "import" + table: "supply" + transport: + param: "Transport" + description: "" + row: "commodity" + col: "transport" + table: "supply" + trade: + param: "Trade" + description: "" + row: "commodity" + col: "trade" + table: "supply" + tax: + param: "Tax" + description: "" + row: "commodity" + col: "tax" + table: "supply" + + intermediate_demand: + param: "Intermediate_Demand" + description: "" + row: "commodity" + col: "sector" + table: "use" + + labor_demand: + param: "Labor_Demand" + description: "" + row: "labor_demand" + col: "sector" + table: "value_added" + capital_demand: + param: "Capital_Demand" + description: "" + row: "capital_demand" + col: "sector" + table: "value_added" + + output_tax: + param: "Output_Tax" + description: "" + row: "output_tax" + col: "sector" + table: "value_added" + + personal_consumption: + param: "Personal_Consumption" + description: "" + row: "commodity" + col: "personal_consumption" + table: "use" + + investment_final_demand: + param: "Investment_Final_Demand" + description: "" + row: "commodity" + col: "investment_final_demand" + table: "use" + + export: + param: "Export" + description: "" + row: "commodity" + col: "export" + table: "use" + + reexport: + param: "Reexport" + description: "" + row: "commodity" + col: "reexport" + table: "use" + + government_final_demand: + param: "Government_Final_Demand" + description: "" + row: "commodity" + col: "government_final_demand" + table: "use" + + +composite_parameters: + Value_Added: + description: "Value added" + elements: + - "labor_demand" + - "capital_demand" + Final_Demand: + description: "Final demand" + elements: + - "personal_consumption" + - "investment_final_demand" + - "government_final_demand" + - "export" + Other_Final_Demand: + description: "Non-export components of final demand" + elements: + - "personal_consumption" + - "investment_final_demand" + - "government_final_demand" + Use: + description: "Use (or input) sections of the IO table" + elements: + - "intermediate_demand" + - "labor_demand" + - "capital_demand" + - "output_tax" + - "personal_consumption" + - "investment_final_demand" + - "government_final_demand" + - "export" + - "margin_supply" + Supply: + description: "Supply (or output) sections of the IO table" + elements: + - "intermediate_supply" + - "import" + - "margin_demand" + - "tax" + + Sector_Subsidy: + description: "Subsidies by sector" + elements: + - "sector_subsidy" + Subsidy: + description: "Total subsidies" + elements: + - "subsidy" + + Duty: + description: "Total duties" + elements: + - "duty" \ No newline at end of file diff --git a/docs/src/data/australia/load_data.md b/docs/src/data/australia/load_data.md new file mode 100644 index 0000000..ceea078 --- /dev/null +++ b/docs/src/data/australia/load_data.md @@ -0,0 +1,35 @@ +# Loading the Data + +To load the Australia data into WiNDC National, you first need to download the necessary tables from the Australian Bureau of Statistics (ABS) as described in the [Data Sources section](@ref australia_data_sources). Be sure to convert the downloaded files to `.xlsx`. + +An example YAML file can be [downloaded here](./australia_local.yaml), for a discussion on how this file was created [read the WiNDC blog on the topic](https://windc.wisc.edu/posts/12/). Update the `paths` section of the YAML file to point to the downloaded data files. In other words, replace +```yaml + paths: + use: + path: path/to/use/data.xlsx + supply: + path: path/to/supply/data.xlsx + value_added: + path: path/to/value_added/data.xlsx +``` +with the actual paths to the downloaded files. + +## Modifying Years + +The years section of the YAML file looks like: +```yaml + years: + "2021-22": 2022 + "2022-23": 2023 + "2023-24": 2024 +``` +The left hand side is the name of the sheet in the Excel file, and the right hand side is the year that will be used in WiNDC National. Modify this section if you are using different years or sheets. + +## Example Usage + +To load the data using the YAML file, use the following code in Julia: +```julia +australia_raw = build_australia_table("australia.yaml") # Change to your YAML file path + +australia,M = calibrate(australia_raw) +``` \ No newline at end of file diff --git a/docs/src/data/united_states/build_local.md b/docs/src/data/united_states/build_local.md new file mode 100644 index 0000000..f08aba5 --- /dev/null +++ b/docs/src/data/united_states/build_local.md @@ -0,0 +1,35 @@ +# Building the Data Locally + +By default the function [`build_us_table`](@ref) will download the latest data from the BEA website. However, this isn't always ideal, you may not have access to the internet or want to save data to prevent later breaking changes. In these cases you can build the data locally by downloading the necessary files yourself and providing the file paths to the function. + +## Download Data + +Obviously, the first step to download the data and save it locally. This links to the necessary data are in the [`Data Sources`](@ref data_sources) section of the United States Data. + +## Configuring the YAML file + +The default YAML files are [`summary.yaml`](summary.yaml) and [`detailed.yaml`](detailed.yaml). Download one of these files. In the `metadata` section, replace the sections + +```yaml +download_url_common: "https://apps.bea.gov/industry/release/zip/SUPPLY-USE.zip" +download: + supply: + pattern: "^Supply_Summary.xlsx$" + use: + pattern: "^Use_Summary.xlsx$" +``` +with +```yaml +paths: + supply: + path: path/to/supply/file.xlsx + use: + path: path/to/use/file.xlsx +``` +where `path/to/supply/file.xlsx` and `path/to/use/file.xlsx` are the paths to the files you downloaded in the previous step. + +Load the data using the modified YAML file. + +```julia +summary_raw = build_us_table("path/to/modified/summary.yaml") +``` diff --git a/docs/src/data_build.md b/docs/src/data/united_states/data_build.md similarity index 62% rename from docs/src/data_build.md rename to docs/src/data/united_states/data_build.md index f274165..8187368 100644 --- a/docs/src/data_build.md +++ b/docs/src/data/united_states/data_build.md @@ -1,11 +1,12 @@ -# Data Build Process +# Building the Data +This page describes the process to build the US National data. The function to build the data is [`build_us_table`](@ref). The only necessary data for the national model is the BEA Supply and Use tables. These tables have the following structure: -![BEA Supply and Use Tables](images/sut.png) +![BEA Supply and Use Tables](./images/sut.png) -Note that the `Other_Final_Demand` region is a collection of multiple columns. However, in the national model we don't distinguish between the values so they are all treated as a single category. The data does not get aggregated together, retrieving that parameter returns all the raw data. +Note that the parameters `Investment_Final_Demand` and `Government_Final_Demand`are collections of multiple columns. However, in the national model we don't distinguish between the values so they are all treated as a single category. The data does not get aggregated together, retrieving that parameter returns all the raw data. Coming from the BEA, this data is not suitable for use in our CGE models. In the following sections we will outline the steps taken to transform the raw data into a format suitable for our models. @@ -17,7 +18,7 @@ The first step to flip the sign of all the values in the Use table. We consider We want all values in the `Intermediate` tables to have the same sign as their flow. For example, in the 2023 Use table the sector `111CA` (Farms) and commodity `Used` (Scrap, Used and secondhand goods) has a value of `18`. We expect all values in the Use table to be negative. Note, this is after flipping the sign from the previous step. In the Use table that value is -18. -To solve this issue we reverse the flow of good with the wrong sign in `Intermediate_Demand` and `Intermediate_Supply`. The `18` moves from Use to be `18` in Supply in the same commodity/sector. +To solve this issue we reverse the flow of good with the wrong sign in `Intermediate_Demand` and `Intermediate_Supply`. The `18` moves from Use to be `18` in Supply in the same commodity/sector. This is done using the function [`WiNDCNational.adjust_intermediate_flows`](@ref). ## Redistribute `CIF/FOB` Adjustments @@ -27,17 +28,17 @@ The `CIF/FOB` column is used to adjust `Transport` margins and `Imports`. The `C - `5241XX` (Insurance carriers, except direct life) - `524200` (Insurance agencies, brokerages, and related activities) -The `CIF/FOB` column is then removed from our data. +The `CIF/FOB` column is then removed from our data. This is done using the function [`WiNDCNational.redistribute_cif_fob`](@ref). ## Disaggregation of `Trade` and `Transport` The two marginal columns `Trade` and `Transport` are transformed into two parameters, `Margin_Demand` and `Margin_Supply`. The positive values from `Trade` and `Transport` become `Margin_Demand` and the negative values become `Margin_Supply`. We do not adjust the signs of these values, `Margin_Supply` acts like an input which means it should be negative. -We also create a new set `margin` which includes the NAICS codes for `Trade` and `Transport`. This set is the column domain of both `Margin_Demand` and `Margin_Supply`. +We also create a new set `margin` which includes the NAICS codes for `Trade` and `Transport`. This set is the column domain of both `Margin_Demand` and `Margin_Supply`. This is done using the function [`WiNDCNational.create_margin_categories`](@ref). ## `Personal_Consumption` and `Household_Supply` -The `Personal_Consumption` column has a mix of positive and negative values. This could should only contain negative values, as it is on the input side. The `Household_Supply` column is created from the positive values. +The `Personal_Consumption` column has a mix of positive and negative values. This could should only contain negative values, as it is on the input side. The `Household_Supply` column is created from the positive values. This is done using the function [`WiNDCNational.create_pce_categories`](@ref). ## `Sector_Subsidy` and `Subsidy` @@ -45,11 +46,30 @@ The `Sector_Subsidy` is a new addition to the Supply/Use framework. It was intro This is in contrast to the `Subsidy` column which, direct from the BEA, is negative. Which is the correct sign so we make no adjustments. +There is no associated function for this step, the values come directly from the table. + +## Marginal Commodities + +Any goods that generate only margins should have no tax or subsidy associated with them. However, in the 2009 Summary Supply table the commodity `441` has a non-zero subsidy value. To fix this we drop any tax/subsidy values associated with marginal commodities. This is done using the function [`WiNDCNational.zero_marginal_tax_subsidy`](@ref). + +## Readjust Negative Value Added + +In the US Supply/Use tables there are some negative capital demands. To fix this we adjust the value added parameters to ensure that all capital demands are non-negative. The adjustment is given by + +```math +\sum_{va} VA(year, va, sector) \cdot \frac{\sum_{year} VA(year, va, sector)}{\sum_{year, va} VA(year, va, sector)} +``` + +This is done using the function [`WiNDCNational.adjust_negative_value_added`](@ref). + + + + # The Final Data Structure The final WiNDC National parameters are given by: -![WiNDC National Parameters](images/sut_calibrated.png) +![WiNDC National Parameters](./images/sut_calibrated.png) These are the actual parameters in the WiNDC National dataset. This means if you want to extract `Export`, you can run ```julia diff --git a/docs/src/data_sources.md b/docs/src/data/united_states/data_sources.md similarity index 67% rename from docs/src/data_sources.md rename to docs/src/data/united_states/data_sources.md index e36a7c0..c65982d 100644 --- a/docs/src/data_sources.md +++ b/docs/src/data/united_states/data_sources.md @@ -6,6 +6,8 @@ We only use one data source for the WiNDC National module, the Bureau of Economi ## Bureau of Economic Analysis - Summary Input/Output Tables -[Link](https://www.bea.gov/industry/input-output-accounts-data) +Download the `SUPPLY-USE` data from the BEA website: + +[BEA Supply/Use table](https://apps.bea.gov/iTable/?reqid=1602&isuri=1&step=8&categories=Core) diff --git a/docs/src/data/united_states/images/sut.asy b/docs/src/data/united_states/images/sut.asy new file mode 100644 index 0000000..49f63df --- /dev/null +++ b/docs/src/data/united_states/images/sut.asy @@ -0,0 +1,96 @@ +import settings; +//settings.prc = false; +settings.outformat="png"; +settings.render = 16; + +size(500,500); + + + + +real width = 8; +real table_width = 100; + +path value_added = (0,-width)--(0,-7*width)--(table_width,-7*width)--(table_width,-width)--cycle; + +draw(value_added); + +label("Labor\_Demand", (50, -1.75*width)); +draw((0, -2.5*width)--(100,-2.5*width), dashed); + +label("Capital\_Demand", (50, -(4+2.5)/2*width)); +draw((0, -4*width)--(100,-4*width), dashed); + +label("Sector\_Subsidy", (50, -(4+5.5)/2*width)); +draw((0, -5.5*width)--(100,-5.5*width), dashed); + +//label("Value Added", (50,-4*width/2)); +label("Other\_Tax", (50,-(5.5+7)/2*width)); + + + +path intermediate_demand = (0,0) -- (0,10*width) -- (table_width,10*width) -- (table_width,0) -- cycle; + +label(rotate(90)*"commoditity", (0, 5*width), align = W); +label("sector", (table_width/2, 10*width), align = N); + + +draw(intermediate_demand); +label("Intermediate\_Demand", (50,65)); + +real start = table_width + width; +table_width = 5.5*width; + +path final_demand = (start, 0) -- (start, 10*width) -- (start + table_width, 10*width) -- (start + table_width, 0) -- cycle; +draw(final_demand); + +label(rotate(90)*"Investment\_Final\_Demand", (start + table_width/4 - .5*width, 5*width), fontsize(8)); + +draw((start + table_width/4, 0)--(start + table_width/4, 10*width), dashed); +label(rotate(90)*"Government\_Final\_Demand", (start + 2*table_width/4 - .5*width, 5*width), fontsize(8)); + + +draw((start + 2*table_width/4, 0)--(start + 2*table_width/4, 10*width), dashed); +label(rotate(90)*"Personal\_Consumption", (start + 3*table_width/4 - .5*width, 5*width), fontsize(8)); + + +draw((start + 3*table_width/4, 0)--(start + 3*table_width/4, 10*width), dashed); +label(rotate(90)*"Export", (start + 4*table_width/4 - .5*width, 5*width)); + +label("Use", ((start+table_width)/2, 12*width), fontsize(20)); + + + + + + + +real start = start + table_width + 5*width; +table_width = 100; + +label("Supply", ((start + start+table_width+width + 1.5*6*width)/2, 12*width), fontsize(20)); + +label(rotate(90)*"commoditity", (start, 5*width), align = W); +label("sectors", (start + table_width/2, 10*width), align = N); + +path intermediate_supply = (start, 0) -- (start, 10*width) -- (start + table_width, 10*width) -- (start + table_width, 0) -- cycle; +draw(intermediate_supply); + +label("Intermediate\_Supply", (start + table_width/2, 65)); + +real start = start + table_width + width; +table_width = 1.5*7*width; + +draw((start, 0)--(start, 10*width)--(start + table_width, 10*width)--(start + table_width, 0)--cycle); + +for(int i = 1; i < 7; ++i) { + draw((start + 1.5*i*width, 0)--(start + 1.5*i*width, 10*width), dashed); +} + +string[] labels = {"CIF/FOB","Import", "Transport", "Trade", "Duty", "Tax", "Subsidy"}; + +for(string p : labels) { + label(rotate(90)*p, (start + 1.5*width/2, 5*width)); + start = start + 1.5*width; +} + diff --git a/docs/src/data/united_states/images/sut.png b/docs/src/data/united_states/images/sut.png new file mode 100644 index 0000000..7f678ef Binary files /dev/null and b/docs/src/data/united_states/images/sut.png differ diff --git a/docs/src/data/united_states/images/sut_calibrated.asy b/docs/src/data/united_states/images/sut_calibrated.asy new file mode 100644 index 0000000..2694f26 --- /dev/null +++ b/docs/src/data/united_states/images/sut_calibrated.asy @@ -0,0 +1,92 @@ +import settings; +//settings.prc = false; +settings.outformat="png"; +settings.render = 16; + +size(500,500); + + + + +real width = 8; +real table_width = 100; + +path value_added = (0,-width)--(0,-7*width)--(table_width,-7*width)--(table_width,-width)--cycle; + +draw(value_added); + +label("Labor\_Demand", (50, -1.75*width)); +draw((0, -2.5*width)--(100,-2.5*width), dashed); + +label("Capital\_Demand", (50, -(4+2.5)/2*width)); +draw((0, -4*width)--(100,-4*width), dashed); + +label("Sector\_Subsidy", (50, -(4+5.5)/2*width)); +draw((0, -5.5*width)--(100,-5.5*width), dashed); + +//label("Value Added", (50,-4*width/2)); +label("Other\_Tax", (50,-(5.5+7)/2*width)); + + + +path intermediate_demand = (0,0) -- (0,10*width) -- (table_width,10*width) -- (table_width,0) -- cycle; + +label(rotate(90)*"commoditity", (0, 5*width), align = W); +label("sector", (table_width/2, 10*width), align = N); + + +draw(intermediate_demand); +label("Intermediate\_Demand*", (50,65)); + +real start = table_width + width; +table_width = 5.5*width; + +path final_demand = (start, 0) -- (start, 10*width) -- (start + table_width, 10*width) -- (start + table_width, 0) -- cycle; +draw(final_demand); + +label(rotate(90)*"Investment\_Final\_Demand", (start + table_width/4 - .5*width, 5*width), fontsize(8)); + +draw((start + table_width/4, 0)--(start + table_width/4, 10*width), dashed); +label(rotate(90)*"Government\_Final\_Demand", (start + 2*table_width/4 - .5*width, 5*width), fontsize(8)); + + +draw((start + 2*table_width/4, 0)--(start + 2*table_width/4, 10*width), dashed); +label(rotate(90)*"Personal\_Consumption*", (start + 3*table_width/4 - .5*width, 5*width), fontsize(8)); + + +draw((start + 3*table_width/4, 0)--(start + 3*table_width/4, 10*width), dashed); +label(rotate(90)*"Export", (start + 4*table_width/4 - .5*width, 5*width)); + +label("Use", ((start+table_width)/2, 12*width), fontsize(20)); + + + +real start = start + table_width + 5*width; +table_width = 100; + +label("Supply", ((start + start+table_width+width + 1.5*6*width)/2, 12*width), fontsize(20)); + +label(rotate(90)*"commoditity", (start, 5*width), align = W); +label("sector", (start + table_width/2, 10*width), align = N); + +path intermediate_supply = (start, 0) -- (start, 10*width) -- (start + table_width, 10*width) -- (start + table_width, 0) -- cycle; +draw(intermediate_supply); + +label("Intermediate\_Supply*", (start + table_width/2, 65)); + +real start = start + table_width + width; +table_width = 1.5*7*width; + +draw((start, 0)--(start, 10*width)--(start + table_width, 10*width)--(start + table_width, 0)--cycle); + +for(int i = 1; i < 7; ++i) { + draw((start + 1.5*i*width, 0)--(start + 1.5*i*width, 10*width), dashed); +} + +string[] labels = {"Household\_Supply*","Import*", "Margin\_Demand*", "Margin\_Supply*", "Duty", "Tax", "Subsidy"}; + +for(string p : labels) { + label(rotate(90)*p, (start + 1.5*width/2, 5*width)); + start = start + 1.5*width; +} + diff --git a/docs/src/data/united_states/images/sut_calibrated.png b/docs/src/data/united_states/images/sut_calibrated.png new file mode 100644 index 0000000..02dc328 Binary files /dev/null and b/docs/src/data/united_states/images/sut_calibrated.png differ diff --git a/docs/src/sets.jl b/docs/src/data/united_states/sets.jl similarity index 97% rename from docs/src/sets.jl rename to docs/src/data/united_states/sets.jl index c2e92bc..e14ca94 100644 --- a/docs/src/sets.jl +++ b/docs/src/data/united_states/sets.jl @@ -73,7 +73,7 @@ elements(summary, :Value_Added) # ## Composite Parameters -# Some parameters are require aggregations and can't be extracted directly. Here +# Some parameters require aggregations and can't be extracted directly. Here # is a full list of these parameters: # # - [`zero_profit`](@ref) diff --git a/docs/src/images/sut.png b/docs/src/images/sut.png deleted file mode 100644 index 03c23f9..0000000 Binary files a/docs/src/images/sut.png and /dev/null differ diff --git a/docs/src/images/sut_calibrated.png b/docs/src/images/sut_calibrated.png deleted file mode 100644 index 232a68f..0000000 Binary files a/docs/src/images/sut_calibrated.png and /dev/null differ diff --git a/docs/src/index.md b/docs/src/index.md index 9403912..c399935 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -14,7 +14,7 @@ using WiNDCNational using DataFrames import MPSGE -summary = build_national_table(:summary) # replace :summary with :detailed for the detailed tables +summary = build_us_table(:summary) # replace :summary with :detailed for the detailed tables zero_profit(summary) # Checking zero profit, not all values are zero market_clearance(summary) diff --git a/docs/src/model/images/A_sector.asy b/docs/src/model/images/A_sector.asy new file mode 100644 index 0000000..bd4a658 --- /dev/null +++ b/docs/src/model/images/A_sector.asy @@ -0,0 +1,64 @@ +import settings; +//settings.prc = false; +settings.outformat="png"; +settings.render = 16; + +size(450,450); + + +label("$A[c=\rm{commodities}]$", (0,0)); +label("$t=2$", (0,0), 1.8*N); +label("$s=0$", (0,0), 1.8*S); + + +//outputs +picture pa_box; + +label(pa_box, "$PA[c]$, ${\rm Armington\_Supply}[c]$", (0,0)); +label(pa_box, "$RA \Rightarrow {\rm Absorption\_Tax}[c]$", (0,0), 2*S); +label(pa_box, "ref\_price = $1-{\rm initial\_absorption\_tax}$", (0,0), 6*S); + +pair PA = (5,5); +draw( (0,0)--PA, Arrow, Margin(8,13) ); +add(pa_box, PA); + + +picture pfx_out; + +label(pfx_out, "$PFX$, ${\rm Export}[c]$", (0,0)); +pair PFX = (-5,5); +draw( (0, 0)--PFX, Arrow, Margin(8,4)); +add(pfx_out, PFX); + + + +//inputs + +picture pm_box; +label(pm_box, "$PM[m = {\rm margins}]$, ${\rm Margin\_Demand}[c,s]$", (0,0)); + +picture pfx_in; +label(pfx_in, "$PFX$, ${\rm Import}[c]$", (0,0)); +label(pfx_in, "$RA\Rightarrow {\rm Import\_Tariff}[c]$", (0,0), 2*S); +label(pfx_in, "ref\_price = $1+{\rm initial\_import\_tariff}$", (0,0), 6*S); + +picture PY_box; +label(PY_box, "$PY[c]$, ${\rm Gross\_Output}[c]$", (0,0)); + + +pair PM = (-5, -5); +pair DM_nest = (5, -5); +pair PY = (9, -10); +pair PFX = (1, -10); + +draw( PM -- (0, 0), Arrow, Margin(5,8) ); +add(pm_box, PM); + +draw( DM_nest -- (0,0), Arrow, Margin(2,8) ); +label("$dm = 2$", DM_nest); + +draw( PFX -- DM_nest, Arrow, Margin(5,2) ); +add(pfx_in, PFX); + +draw( PY -- DM_nest, Arrow, Margin(5,2) ); +add(PY_box, PY); \ No newline at end of file diff --git a/docs/src/model/images/A_sector.png b/docs/src/model/images/A_sector.png new file mode 100644 index 0000000..92b320e Binary files /dev/null and b/docs/src/model/images/A_sector.png differ diff --git a/docs/src/model/images/ms_sector.asy b/docs/src/model/images/ms_sector.asy new file mode 100644 index 0000000..baec497 --- /dev/null +++ b/docs/src/model/images/ms_sector.asy @@ -0,0 +1,33 @@ +import settings; +//settings.prc = false; +settings.outformat="png"; +settings.render = 16; + +size(450,200); + + +label("$MS[m=\rm{margins}]$", (0,0)); +label("$t=0$", (0,0), 1.8*N); +label("$s=0$", (0,0), 1.8*S); + + +//outputs +picture pm_box; + +label(pm_box, "$PM[m]$, $\displaystyle\sum_{c\in{\rm commodities}} {\rm Margin\_Supply}[c,m]$", (0,0)); + +pair PM = (0,3); +draw( (0,0)--PM, Arrow, Margin(5,8) ); +add(pm_box, PM); + + +//inputs + +picture py_box; +label(py_box, "$PY[c = {\rm commodities}]$, ${\rm Margin\_Supply}[c,m]$", (0,0)); + +pair PY = (0,-3); + +draw( PY -- (0, 0), Arrow, Margin(5,8) ); +add(py_box, PY); + diff --git a/docs/src/model/images/ms_sector.png b/docs/src/model/images/ms_sector.png new file mode 100644 index 0000000..9591569 Binary files /dev/null and b/docs/src/model/images/ms_sector.png differ diff --git a/docs/src/model/images/national.asy b/docs/src/model/images/national.asy new file mode 100644 index 0000000..bfca442 --- /dev/null +++ b/docs/src/model/images/national.asy @@ -0,0 +1,37 @@ +import settings; +//settings.prc = false; +settings.outformat="png"; +settings.render = 16; + +real image_size = 450; +real Scale = .5; + + + + +include "y_sector.asy"; +picture y_sector = currentpicture; +currentpicture = new picture; + +include "A_sector.asy"; +picture a_sector = currentpicture; +currentpicture = new picture; + +include "ms_sector.asy"; +picture ms_sector = currentpicture; +currentpicture = new picture; + +include "ra_consumer.asy"; +picture ra_consumer = currentpicture; +currentpicture = new picture; + + +//size(500,500); + +add(scale(Scale)*y_sector.fit(), (0,0)); +add(scale(Scale)*a_sector.fit(), (Scale*image_size,0)); + +add(scale(Scale)*ms_sector.fit(), (0,-Scale*image_size)); +add(scale(Scale)*ra_consumer.fit(), (Scale*image_size,-Scale*image_size)); + + diff --git a/docs/src/model/images/national.png b/docs/src/model/images/national.png new file mode 100644 index 0000000..170e235 Binary files /dev/null and b/docs/src/model/images/national.png differ diff --git a/docs/src/model/images/ra_consumer.asy b/docs/src/model/images/ra_consumer.asy new file mode 100644 index 0000000..49bc779 --- /dev/null +++ b/docs/src/model/images/ra_consumer.asy @@ -0,0 +1,48 @@ +import settings; +//settings.prc = false; +settings.outformat="png"; +settings.render = 16; + +size(450,450); + + +label("$RA$", (0,0)); + + +//outputs +picture py_box; +label(py_box, "$PY[c={\rm commodities}]$", (0,0)); +label(py_box, "${\rm Household\_Supply}[c]$", (0,0), 2*S); +pair PY = (-8,5); +draw( (0,0)--PY, Arrow, Margin(5,12)); +add(py_box, PY); + +picture PFX_box; +label(PFX_box, "$PFX$", (0,0)); +label(PFX_box, "${\rm Balance\_of\_Payments}$", (0,0), 2*S); +pair PFX_out = (-4,8); +draw( (0, 0)--PFX_out, Arrow, Margin(5,8)); +add(PFX_box, PFX_out); + +picture PA_out; +label(PA_out, "$PA[c={\rm commodities}]$", (0,0)); +label(PA_out, "$-{\rm Other\_Final\_Demand}$", (0,0), 2*S); +pair PA = (4,8); +draw( (0,0) -- PA, Arrow, Margin(5,8) ); +add(PA_out, PA); + +picture PVA_box; +label(PVA_box, "$PVA[va={\rm value\_added}]$", (0,0)); +label(PVA_box, "$\displaystyle \sum_{s\in {\rm sectors}} {\rm Value_\_Added}[va,s]$", (0,0), 2*S); +pair PVA = (8,5); +draw((0,0) -- PVA, Arrow, Margin(5,20) ); +add(PVA_box, PVA); + + +//inputs + +picture pa_in; +label(pa_in, "$PA[c = {\rm commodities}]$, ${\rm Personal\_Consumption}[c]$", (0,0)); +pair PA_in = (0,-5); +draw( PA_in -- (0, 0), Arrow, Margin(5,5) ); +add(pa_in, PA_in); \ No newline at end of file diff --git a/docs/src/model/images/ra_consumer.png b/docs/src/model/images/ra_consumer.png new file mode 100644 index 0000000..7d3ce75 Binary files /dev/null and b/docs/src/model/images/ra_consumer.png differ diff --git a/docs/src/model/images/y_sector.asy b/docs/src/model/images/y_sector.asy new file mode 100644 index 0000000..62f46f5 --- /dev/null +++ b/docs/src/model/images/y_sector.asy @@ -0,0 +1,44 @@ +import settings; +//settings.prc = false; +settings.outformat="png"; +settings.render = 16; + +size(450,450); + + +label("$Y[s=\rm{sectors}]$", (0,0)); +label("$t=0$", (0,0), 1.8*N); +label("$s=0$", (0,0), 1.8*S); + + +//outputs +picture py_box; + +label(py_box, "$PY[c={\rm commodities}]$, ${\rm Intermediate\_Supply}[c,s]$", (0,0)); +label(py_box, "$RA \Rightarrow {\rm Output\_Tax}[s]$", (0,0), 2*S); + +pair PY = (0,5); +draw( (0,0)--PY, Arrow, Margin(5,8) ); +add(py_box, PY); + + +//inputs + +picture pa_box; +label(pa_box, "$PA[c = {\rm commodities}]$, ${\rm Intermediate\_Demand}[c,s]$", (0,0)); + +picture va_box; +label(va_box, "$PVA[va={\rm value\_added}]$, ${\rm Value\_Added}[va,s]$", (0,0)); + +pair PA = (-5,-5); +pair VA_nest = (5,-5); +pair VA = (5,-10); + +draw( PA -- (0, 0), Arrow, Margin(5,8) ); +add(pa_box, PA); + +draw( VA_nest -- (0,0), Arrow, Margin(2,8) ); +label("$va = 1$", VA_nest); + +draw( VA -- VA_nest, Arrow, Margin(5,2) ); +add(va_box, VA); \ No newline at end of file diff --git a/docs/src/model/images/y_sector.png b/docs/src/model/images/y_sector.png new file mode 100644 index 0000000..a2ff033 Binary files /dev/null and b/docs/src/model/images/y_sector.png differ diff --git a/docs/src/model/overview.jl b/docs/src/model/overview.jl new file mode 100644 index 0000000..732b14d --- /dev/null +++ b/docs/src/model/overview.jl @@ -0,0 +1,158 @@ +# # Model Overview + +# The WiNDC National Model is written in [MPSGE](https://github.com/julia-mpsge/MPSGE.jl), +# a high level language for writing computable general equilibrium models. The +# syntax allows a modeler to focus on the economic structure of the model +# rather than the mathematical details of its implementation. + +# For the full model specification, see the documentation: [`national_mpsge`](@ref). +# This document will construct the model step by step, explaining each component +# along the way. + +# ## Building the Model + +# The first step is to load the data and filter to just the year we want to model, +# in this case 2024. This step isn't strictly necessary, as the model can be built +# for multiple years at once, but the years don't interact so it's really just a +# sequence of models. We will also calibrate the data, suppressing the output. + +using WiNDCNational +using MPSGE +import DataStructures: DefaultDict + +summary_raw_all_years = build_us_table(); +summary_raw = National( + table(summary_raw_all_years, :year => 2024), + sets(summary_raw_all_years), + elements(summary_raw_all_years); +) +summary, _ = calibrate(summary_raw, silent = true); + +# Let's also break out a few sets we'll need later. This is primarily for convenience. + +sectors = elements(summary, :sector) |> x -> x[!,:name]; +commodities = elements(summary, :commodity) |> x -> x[!,:name]; +margins = elements(summary, :margin) |> x -> x[!,:name]; +value_added = elements(summary, :Value_Added; base=true) |> x -> x[!,:name]; +final_demand = elements(summary, :Final_Demand; base=true) |> x -> x[!,:name]; + +exports = elements(summary, :export) |> x -> x[!,:name]; +imports = elements(summary, :import) |> x -> x[!,:name]; +PCE = elements(summary, :personal_consumption) |> x -> x[!,:name]; +HHS = PCE; + +# Now we can start building the model. The first step is to create the MPSGE model object. + +M = MPSGEModel() + +# The model starts empty, we add variables and parameters in the next steps. + +# ### Parameters + +# The model has three available parameters: +# 1. Absorption_Tax +# 2. Import_Tariff +# 3. Output_Tax +# with the expected initial values. We've written functions to extract these values +# from `summary` as DataFrames. We can combine them into a single DataFrame, convert +# this to a default dictionary, and pass it to the `@parameters` macro. + +vcat( + absorption_tax_rate(summary; minimal=false), + import_tariff_rate(summary; minimal=false), + output_tax_rate(summary; minimal=false) +) |> +x -> DefaultDict(0, Dict((row[:row], row[:col]) => row[:value] for row in eachrow(x))) |> +x -> @parameters(M, begin + Absorption_Tax[c=commodities], x[c, :atr] + Import_Tariff[c=commodities], x[c, :itr] + Output_Tax[s=sectors], x[:otr, s] +end); + +# The output is suppressed, but we can view the parameters in the model just by +# viewing the model: + +M + +# ### Sectors + +# The national model follows a standard Armington structure with three sectors +# corresponding to Production, Supply, and Margins. We can add these sectors to the model +# using the `@sectors` macro. + +@sectors(M, begin + Y[s=sectors], (description = "Sectoral Production") + A[c=commodities], (description = "Armington Supply") + MS[m=margins], (description = "Margin Supply") +end); + +M + +# Notice in the model output, the sectors are displayed with their descriptions. +# The same would apply for parameters, but the parameter names are self-descriptive. + +# ### Commodities + +@commodities(M, begin + PA[c=commodities], (description = "Armington Price") + PY[c=commodities], (description = "Output Price") + PVA[va=value_added], (description = "Value Added Price") + PM[m=margins], (description = "Margin Price") + PFX, (description = "Foreign Exchange") +end) + +# ### Consumer + +# The model has only a single consumer, the representative agent. In this case +# we add the consumer using the `@consumer` macro. Notice this is singular, both +# versions exist for each of the variable macros. + +@consumer(M, RA, description = "Representative Agent") + +M + +# ### Y Production + +WiNDCNational.sectoral_production_data(summary; output=:Dict) |> x -> +@production(M, Y[s=sectors], [t=0,s=0, va => s=1], begin + @output(PY[c=commodities], x[c, s, :intermediate_supply], t, taxes = [Tax(RA, M[:Output_Tax][s])]) + @input(PA[c=commodities], x[c, s, :intermediate_demand], s) + @input(PVA[va = value_added], x[va, s, :value_added], va) +end) + +# ### A Production + +WiNDCNational.armington_supply_data(summary; output=:Dict) |> x -> +@production(M, A[c=commodities], [t=2, s=0, dm=>s=2], begin + @output(PA[c], x[c, :as, :armington_supply], t, taxes = [Tax(RA, M[:Absorption_Tax][c])], reference_price = 1-x[c,:atr, :absorption_tax_rate]) + @output(PFX, sum(x[c, e, :export] for e in exports), t) + @input(PM[m=margins], x[c, m, :margin_demand], s) + @input(PFX, sum(x[c, i, :import] for i in imports), dm, taxes = [Tax(RA, M[:Import_Tariff][c])], reference_price = 1+x[c, :itr, :import_tariff_rate]) + @input(PY[c], x[c, :go, :gross_output], dm) +end) + + +# ### MS Production + + +WiNDCNational.margin_supply_data(summary; output=:Dict) |> x -> +@production(M, MS[m=margins], [t=0, s=0], begin + @output(PM[m], sum(x[c, m] for c in commodities if x[c, m] != 0), t) + @input(PY[c=commodities], x[c,m], s) +end) + +# ### RA Demand + +WiNDCNational.representative_agent_data(summary; output=:Dict) |> x -> +@demand(M, RA, begin + @final_demand(PA[c=commodities], sum(x[c, pce, :personal_consumption] for pce in PCE)) + @endowment(PY[c=commodities], sum(x[c, hhs, :household_supply] for hhs in HHS)) + @endowment(PFX, x[:bop, :bop, :balance_of_payments]) + @endowment(PA[c=commodities], -x[c, :ofd, :ofd]) + @endowment(PVA[va=value_added], sum(x[va, s, :value_added] for s∈sectors)) +end) + +# ## Verifying the Calibration + +solve!(M; cumulative_iteration_limit=0) + diff --git a/docs/src/model/overview.md b/docs/src/model/overview.md new file mode 100644 index 0000000..eb6bd31 --- /dev/null +++ b/docs/src/model/overview.md @@ -0,0 +1,49 @@ +# Overview + +The WiNDC National Model is written in [MPSGE](https://github.com/julia-mpsge/MPSGE.jl), a high level language for writing computable general equilibrium models. The syntax allows a modeler to focus on the economic structure of the model rather than the mathematical details of its implementation. For the full model specification, see the documentation: [`national_mpsge`](@ref). + +## Model Diagram + +The following tree diagram represents the structure of the WiNDC National Model: + +![Model Structure](./images/national.png) + +This diagram encapsulates all information for implementing the model, either in MPSGE or as equations in an MCP formulation. Take, for example, the $Y$ sector. In the center of the tree we see, +```math +Y[s = {\rm sectors}] +``` +This indicates this production sector is indexed over the set of sectors. Above we see $t=0$ and below $s=0$, these indicate the elasticities of transformation and substitution, respectively. The arrows indicate the flow of goods, arrows pointing toward the center are inputs, and arrows pointing away from the center are outputs. The final non-leaf node is $va=1$, indicating a nest with elasticity of substitution equal to 1. + +Each leaf of the tree contains information to calculate its price and quantity. For example, consider the $PFX$ input to the $A$ sector. The adjusted price of this input is given by, +```math +\frac{PFX\cdot(1+{\rm Import\_Tariff}[c])}{1+{\rm initial\_import\_tariff}} +``` +The initial $PFX$ price is a basic price, adding the tax adjusts it to a producer price. Since this quantity is an import, it comes from the Supply table and is calibrated in basic price. The purpose of the reference price in the denominator is to ensure that at the benchmark equilibrium, the adjusted price equals the initial price. This ensures that the benchmark equilibrium is a solution to the model. + +We also need to adjust the reference quantity into producer price. We do this by multiplying by the reference price, +```math +{\rm Import}[c] \cdot (1+{\rm initial\_import\_tariff}) +``` +Notice the total value of this input at the benchmark equilibrium is the producer price of imports: +```math +\begin{align*} +\frac{PFX\cdot(1+{\rm Import\_Tariff}[c])}{1+{\rm initial\_import\_tariff}} \cdot {\rm Import}[c] \cdot (1+{\rm initial\_import\_tariff}) &= PFX \cdot (1+{\rm Import\_Tariff}[c]) \cdot {\rm Import}[c] \\ +& = {\rm Total\_Value\_of\_Imports}[c] + {\rm Import\_Tariff\_Revenue}[c] +\end{align*} +``` + +The same idea holds for the consumer $RA$, except that outputs represent endowments and inputs represent final demands. + +## Social Accounting Matrix + +This can be converted to a traditional social accounting matrix by taking the sectors and consumers on the rows and the commodities on the columns: + +| | $Y[s]$ | $A[c]$ | $MS[m]$ | $RA$ | +|:---: |:---: |:---: |:---: |:---:| +| $PY[c]$ | Int_Sup[c,s] | -Gro_Out[c] | -Mar_Sup[c,m] | HS[c] | +| $PA[c]$ | -Int_Dem[c,s] | Arm_Sup[c] | | -Other\_FD - PCE[c] | +| $PM[m]$ | | -Mar_Dem[c,m] | $\sum {\rm Mar\_Sup}[c,m]$ | | +| $PFX$ | | Exp[c]-Imp[c] | | BOP | +| $PVA[va]$ | -Val_Add[va,s] | | | $\sum {\rm Val\_Add}[va,s]$ | + +However, doing so will obscure the nested structure of the production sectors, hide the elasticities, and require additional discussion to allocate taxes. We also shortened the names of the input data for readability. Again, this would require additional explanation, whereas the tree version has the full names and structure visible. \ No newline at end of file diff --git a/src/australia/build.jl b/src/australia/build.jl index df9b7c9..fdfefec 100644 --- a/src/australia/build.jl +++ b/src/australia/build.jl @@ -161,16 +161,11 @@ function merge_export_reexport(X::AustraliaNational; kwargs...) SETS = SETS |> x -> subset(x, :name => ByRow(!∈([:Reexport])) - ) #|> - #x -> push!(x, (name = :Household_Supply, description = "Positive values in PCE (negative in USE table)", domain = :parameter)) |> - #x -> push!(x, (name = :Personal_Consumption, description = "Negative values in PCE (positive in USE table)", domain = :parameter)) - + ) ELEMENTS = ELEMENTS |> x -> subset(x, :set => ByRow(!∈([:Reexport])) - ) #|> - #x -> push!(x, (name = :household_supply, description = "Household Supply", set = :Household_Supply)) |> - #x -> push!(x, (name = :personal_consumption, description = "Personal Consumption", set = :Personal_Consumption)) + ) return AustraliaNational(DATA, SETS, ELEMENTS; regularity_check=true) diff --git a/src/build.jl b/src/build.jl index 847c3cf..1483be4 100644 --- a/src/build.jl +++ b/src/build.jl @@ -1,5 +1,3 @@ - - function parse_excel_set_elements(data::Matrix{Any}) 1∈size(data) || error("Must be either a column or row vector") return vec(data) @@ -9,6 +7,7 @@ function parse_excel_set_elements(data::String) return [data] end +parse_excel_set_elements(::Missing) = missing struct ExcelRange @@ -98,8 +97,9 @@ Common: - years: A dictionary of years with keys the sheet names and value the year `Dict{String, Int}`. - `transformation_keywords`: Used to pass parameters to the data transformation -functions. In particular, it's useful to have an `insurance_codes` keyword for -the redistribution of `CIF_FOB` data to imports and transport. +functions. For the United States national tables, we use two transformations: + - `insurance_codes`: A list of insurance codes for the redistribution of `CIF_FOB` data to imports and transport. + - `marginal_commodities`: A list of marginal commodities to include in the transformation. - `na_values`: A list of values to treat as NA (not available). This is useful for handling missing data in the Excel files. @@ -111,7 +111,8 @@ Each set is listed with the following attributes: - domain - Which column the set operates on - values - An Excel range that corresponds to the NAICS codes - descriptions - An Excel range that corresponds to the descriptions of the NAICS codes -- table - Either "use" or "supply" +- table - Needs to match the listed paths for download names +- value_labels (optional) - A list of labels for the set elements. If not provided, the values from the Excel range will be used. ### parameters @@ -121,7 +122,7 @@ Each parameter name is listed with the following attributes: - description - A description of the parameter - row - The row set of the parameter. - col - The column set of the parameter -- table - Either "use" or "supply" +- table - Needs to match the listed paths for download names - flip_sign - Boolean, changes the sign of the data. Only used on `sector_subsidy` since the values are reported as positive, but must be negative. @@ -245,18 +246,21 @@ function build_national_table(yaml_path::String, output_type::Type{<:AbstractNat # Build sets and elements year = first(keys(years)) # Sets and elements are the same for all years. for (set_name,set) in info["sets"] - X = tables[set["table"]][year] # Create a new set push!(SETS, (name = Symbol(set_name), description = set["description"], domain = Symbol(set["domain"]))) - # Add all the elements - names = parse_excel_set_elements(X[set["values"]]) + # Add all the elements - Perhaps an error should be thrown if both value_labels and values are missing + names = Symbol.(get(set, "value_labels", [])) + if isempty(names) + names = Symbol.(parse_excel_set_elements(X[set["values"]])) + end + ELEMENTS = vcat( ELEMENTS, DataFrame( - name = Symbol.(names), + name = names, description = parse_excel_set_elements(X[set["descriptions"]]), set = repeat([Symbol(set_name)], length(names)) ) @@ -285,8 +289,15 @@ function build_national_table(yaml_path::String, output_type::Type{<:AbstractNat flip_sign = get(parm, "flip_sign", false) # Some parameters may need to flip the sign, `sector_subsidy` is the reason sign = flip_sign ? -sign : sign - row_range = set_info[parm["row"]]["values"] - col_range = set_info[parm["col"]]["values"] + row_names = Symbol.(get(set_info[parm["row"]], "value_labels", [])) + if isempty(row_names) + row_names = Symbol.(parse_excel_set_elements(X[set_info[parm["row"]]["values"]])) + end + + col_names = Symbol.(get(set_info[parm["col"]], "value_labels", [])) + if isempty(col_names) + col_names = Symbol.(parse_excel_set_elements(X[set_info[parm["col"]]["values"]])) + end # Transform set ranges into a rectangular excel range data_range = excel_intersection( @@ -294,9 +305,10 @@ function build_national_table(yaml_path::String, output_type::Type{<:AbstractNat ExcelRange(set_info[parm["col"]]["values"]) ) + new_data = DataFrame( - [Symbol.(parse_excel_set_elements(X[row_range])) X[data_range]], - [:row, Symbol.(parse_excel_set_elements(X[col_range]))...] + [row_names X[data_range]], + [:row, col_names...] ) |> x -> stack(x, Not(:row), variable_name = :col) |> x -> dropmissing(x) |> @@ -305,7 +317,7 @@ function build_national_table(yaml_path::String, output_type::Type{<:AbstractNat ) |> x -> transform(x, :row => ByRow(y -> Symbol(parm_name)) => :parameter, - :row => (y -> Symbol.(y)) => :row, + #:row => (y -> Symbol.(y)) => :row, :col => ByRow(y -> Symbol(y)) => :col, :value => ByRow(y -> sign* (isa(y,String) ? parse(Float64, y) : y) /1_000) => :value, :row => ByRow(y -> year) => :year, @@ -313,6 +325,7 @@ function build_national_table(yaml_path::String, output_type::Type{<:AbstractNat x -> subset(x, :value => ByRow(!=(0)) ) + DATA = vcat(DATA, new_data) end diff --git a/src/model.jl b/src/model.jl index 38d4f79..1dcc591 100644 --- a/src/model.jl +++ b/src/model.jl @@ -1,6 +1,6 @@ """ - national_mpsge(data::T; year = 2023) where T<:AbstractNationalTable + national_mpsge(data::T; year = elements(data, :year) |> x -> maximum(x[!, :name])) where T<:AbstractNationalTable Create a MPSGE model from the given National object. @@ -8,6 +8,10 @@ Create a MPSGE model from the given National object. 1. `data` - A National object. +## Keyword Arguments + +- `year` - The year for which to build the model. Defaults to the maximum year in the data. + ## Output Returns a MPSGEModel object. @@ -36,73 +40,50 @@ Returns a MPSGEModel object. ### Productions -```julia -table(X, - :Intermediate_Supply, - :Intermediate_Demand, - :Labor_Demand, - :Capital_Demand; - normalize = :Use -) |> -x -> DefaultDict(0, Dict((row[:row], row[:col], parameter_col(row[:parameter])) => row[:value] for row in eachrow(x))) |> -x -> -@production(M, Y[s=sectors], [t=0,s=0, va => s=1], begin - @output(PY[c=commodities], x[c, s, :intermediate_supply], t, taxes = [Tax(RA, Output_Tax[s])]) - @input(PA[c=commodities], x[c, s, :intermediate_demand], s) - @input(PVA[va = value_added], x[va, s, :value_added], va) -end) - -table(X, :Margin_Supply; normalize=:Margin_Supply) |> -x -> DefaultDict(0, Dict((row[:row], row[:col]) => row[:value] for row in eachrow(x))) |> -x -> -@production(M, MS[m=margins], [t=0, s=0], begin - @output(PM[m], sum(x[c, m] for c in commodities if x[c, m] != 0), t) - @input(PY[c=commodities], x[c,m], s) -end) +The data functions are: +- [`sectoral_production_data`](@ref) +- [`margin_supply_data`](@ref) +- [`armington_supply_data`](@ref) +- [`representative_agent_data`](@ref) -vcat( - armington_supply(X, minimal=false), - gross_output(X, minimal=false) |> x-> subset(x, :value => ByRow(>(1e-6))), - absorption_tax_rate(X, minimal=false), - import_tariff_rate(X, minimal=false), - table(X, :Export, :Import, :Margin_Demand; normalize = :Use) -) |> -x -> DefaultDict(0, Dict((row[:row], row[:col], row[:parameter]) => row[:value] for row in eachrow(x))) |> -x -> -@production(M, A[c=commodities], [t=2, s=0, dm=>s=2], begin - @output(PA[c], x[c, :as, :armington_supply], t, taxes = [Tax(RA, Absorption_Tax[c])], reference_price = 1-x[c,:atr, :absorption_tax_rate]) - @output(PFX, sum(x[c, e, :export] for e in exports), t) - @input(PM[m=margins], x[c, m, :margin_demand], s) - @input(PFX, sum(x[c, i, :import] for i in imports), dm, taxes = [Tax(RA, Import_Tariff[c])], reference_price = 1+x[c, :itr, :import_tariff_rate]) - @input(PY[c], x[c, :go, :gross_output], dm) -end) - -### Demands -vcat( - table(X, :Personal_Consumption, :Household_Supply, :Value_Added; normalize=:Use), - balance_of_payments(X, minimal=false), - table(X, :Investment_Final_Demand, :Government_Final_Demand; normalize=:Use) |> - x-> groupby(x, [:row, :year]) |> - x -> combine(x, - [:col, :parameter] .=> (y -> (:ofd)) .=> [:col, :parameter], - :value => sum => :value) -) |> -x -> DefaultDict(0, Dict((row[:row], row[:col], parameter_col(row[:parameter])) => row[:value] for row in eachrow(x))) |> -x -> @demand(M, RA, begin - @final_demand(PA[c=commodities], sum(x[c, pce, :personal_consumption] for pce in PCE)) - @endowment(PY[c=commodities], sum(x[c, hhs, :household_supply] for hhs in HHS)) - @endowment(PFX, x[:bop, :bop, :balance_of_payments]) - @endowment(PA[c=commodities], -x[c, :ofd, :ofd]) - @endowment(PVA[va=value_added], sum(x[va, s, :value_added] for s∈sectors)) -end) +```julia + sectoral_production_data(X; output=:Dict) |> x -> + @production(M, Y[s=sectors], [t=0,s=0, va => s=1], begin + @output(PY[c=commodities], x[c, s, :intermediate_supply], t, taxes = [Tax(RA, Output_Tax[s])]) + @input(PA[c=commodities], x[c, s, :intermediate_demand], s) + @input(PVA[va = value_added], x[va, s, :value_added], va) + end) -``` + margin_supply_data(X; output=:Dict) |> x -> + @production(M, MS[m=margins], [t=0, s=0], begin + @output(PM[m], sum(x[c, m] for c in commodities if x[c, m] != 0), t) + @input(PY[c=commodities], x[c,m], s) + end) + + + armington_supply_data(X; output=:Dict) |> x -> + @production(M, A[c=commodities], [t=2, s=0, dm=>s=2], begin + @output(PA[c], x[c, :as, :armington_supply], t, taxes = [Tax(RA, Absorption_Tax[c])], reference_price = 1-x[c,:atr, :absorption_tax_rate]) + @output(PFX, sum(x[c, e, :export] for e in exports), t) + @input(PM[m=margins], x[c, m, :margin_demand], s) + @input(PFX, sum(x[c, i, :import] for i in imports), dm, taxes = [Tax(RA, Import_Tariff[c])], reference_price = 1+x[c, :itr, :import_tariff_rate]) + @input(PY[c], x[c, :go, :gross_output], dm) + end) + representative_agent_data(X; output=:Dict) |> x -> + @demand(M, RA, begin + @final_demand(PA[c=commodities], sum(x[c, pce, :personal_consumption] for pce in PCE)) + @endowment(PY[c=commodities], sum(x[c, hhs, :household_supply] for hhs in HHS)) + @endowment(PFX, x[:bop, :bop, :balance_of_payments]) + @endowment(PA[c=commodities], -x[c, :ofd, :ofd]) + @endowment(PVA[va=value_added], sum(x[va, s, :value_added] for s∈sectors)) + end) +``` """ -function national_mpsge(data::T; year = 2023) where T<:AbstractNationalTable +function national_mpsge(data::T; year = elements(data, :year) |> x -> maximum(x[!, :name])) where T<:AbstractNationalTable X = T( table(data, :year => year), @@ -154,41 +135,23 @@ function national_mpsge(data::T; year = 2023) where T<:AbstractNationalTable @consumer(M, RA, description = "Representative Agent") - parameter_col(parameter) = parameter∈[:labor_demand, :capital_demand] ? :value_added : parameter - table(X, - :Intermediate_Supply, - :Intermediate_Demand, - :Labor_Demand, - :Capital_Demand; - normalize = :Use - ) |> - x -> DefaultDict(0, Dict((row[:row], row[:col], parameter_col(row[:parameter])) => row[:value] for row in eachrow(x))) |> - x -> + + sectoral_production_data(X; output=:Dict) |> x -> @production(M, Y[s=sectors], [t=0,s=0, va => s=1], begin @output(PY[c=commodities], x[c, s, :intermediate_supply], t, taxes = [Tax(RA, Output_Tax[s])]) - @input(PA[c=commodities], x[c, s, :intermediate_demand], s) # Heads up, negative - @input(PVA[va = value_added], x[va, s, :value_added], va) # Heads up, negative + @input(PA[c=commodities], x[c, s, :intermediate_demand], s) + @input(PVA[va = value_added], x[va, s, :value_added], va) end) - table(X, :Margin_Supply; normalize=:Margin_Supply) |> - x -> DefaultDict(0, Dict((row[:row], row[:col]) => row[:value] for row in eachrow(x))) |> - x -> + margin_supply_data(X; output=:Dict) |> x -> @production(M, MS[m=margins], [t=0, s=0], begin - @output(PM[m], sum(x[c, m] for c in commodities if x[c, m] != 0), t) # Heads up, negative + @output(PM[m], sum(x[c, m] for c in commodities if x[c, m] != 0), t) @input(PY[c=commodities], x[c,m], s) end) - vcat( - armington_supply(X, minimal=false), - gross_output(X, minimal=false) |> x-> subset(x, :value => ByRow(>(1e-6))), - absorption_tax_rate(X, minimal=false), - import_tariff_rate(X, minimal=false), - table(X, :Export, :Import, :Margin_Demand; normalize = :Use) - ) |> - x -> DefaultDict(0, Dict((row[:row], row[:col], row[:parameter]) => row[:value] for row in eachrow(x))) |> - x -> + armington_supply_data(X; output=:Dict) |> x -> @production(M, A[c=commodities], [t=2, s=0, dm=>s=2], begin @output(PA[c], x[c, :as, :armington_supply], t, taxes = [Tax(RA, Absorption_Tax[c])], reference_price = 1-x[c,:atr, :absorption_tax_rate]) @output(PFX, sum(x[c, e, :export] for e in exports), t) @@ -198,7 +161,140 @@ function national_mpsge(data::T; year = 2023) where T<:AbstractNationalTable end) - vcat( + representative_agent_data(X; output=:Dict) |> x -> + @demand(M, RA, begin + @final_demand(PA[c=commodities], sum(x[c, pce, :personal_consumption] for pce in PCE)) + @endowment(PY[c=commodities], sum(x[c, hhs, :household_supply] for hhs in HHS)) + @endowment(PFX, x[:bop, :bop, :balance_of_payments]) + @endowment(PA[c=commodities], -x[c, :ofd, :ofd]) + @endowment(PVA[va=value_added], sum(x[va, s, :value_added] for s∈sectors)) + end) + + + return M +end + +""" + sectoral_production_data(data::T; output = :DataFrame) where T<:AbstractNationalTable + +Extract `Intermediate_Demand`, `Intermediate_Supply`, `Labor_Demand`, and `Capital_Demand` from a National object. +Also make `Labor_Demand` and `Capital_Demand` use `value_added` as the parameter column. + +```julia +out = table(data, + :Intermediate_Demand, + :Intermediate_Supply, + :Labor_Demand, + :Capital_Demand; + normalize = :Use +) +``` + +""" +function sectoral_production_data(data::T; output = :DataFrame) where T<:AbstractNationalTable + parameter_col(parameter) = parameter∈[:labor_demand, :capital_demand] ? :value_added : parameter + + out = table(data, + :Intermediate_Demand, + :Intermediate_Supply, + :Labor_Demand, + :Capital_Demand; + normalize = :Use + ) + + if output == :DataFrame + return out + elseif output == :Dict + return DefaultDict(0, Dict((row[:row], row[:col], parameter_col(row[:parameter])) => row[:value] for row in eachrow(out))) + else + error("Unsupported output type: $output") + end +end + + +""" + margin_supply_data(data::T; output = :DataFrame) where T<:AbstractNationalTable + +Extract `Margin_Supply` from a National object. + +```julia +table(data, :Margin_Supply; normalize=:Margin_Supply) +``` +""" +function margin_supply_data(data::T; output = :DataFrame) where T<:AbstractNationalTable + out = table(data, :Margin_Supply; normalize=:Margin_Supply) + + if output == :DataFrame + return out + elseif output == :Dict + return DefaultDict(0, Dict((row[:row], row[:col]) => row[:value] for row in eachrow(out))) + else + error("Unsupported output type: $output") + end +end + +""" + armington_supply_data(X::T; output = :DataFrame) where T<:AbstractNationalTable + +Extract Armington supply related data from a National object, including: +- [`armington_supply`](@ref) +- [`gross_output`](@ref) +- [`absorption_tax_rate`](@ref) +- [`import_tariff_rate`](@ref) +- `Export`, `Import`, and `Margin_Demand` + +```julia +vcat( + armington_supply(X, minimal=false), + gross_output(X, minimal=false) |> x-> subset(x, :value => ByRow(>(1e-6))), + absorption_tax_rate(X, minimal=false), + import_tariff_rate(X, minimal=false), + table(X, :Export, :Import, :Margin_Demand; normalize = :Use) +) +``` +""" +function armington_supply_data(X::T; output = :DataFrame) where T<:AbstractNationalTable + out = vcat( + armington_supply(X, minimal=false), + gross_output(X, minimal=false) |> x-> subset(x, :value => ByRow(>(1e-6))), + absorption_tax_rate(X, minimal=false), + import_tariff_rate(X, minimal=false), + table(X, :Export, :Import, :Margin_Demand; normalize = :Use) + ) + + if output == :DataFrame + return out + elseif output == :Dict + return DefaultDict(0, Dict((row[:row], row[:col], row[:parameter]) => row[:value] for row in eachrow(out))) + else + error("Unsupported output type: $output") + end +end + +""" + representative_agent_data(X::T; output = :DataFrame) where T<:AbstractNationalTable + +Extract representative agent related data from a National object, including: +- `Personal_Consumption`, `Household_Supply`, `Value_Added` +- [`balance_of_payments`](@ref) +- `Investment_Final_Demand`, `Government_Final_Demand` -> Sum values as a new parameter `ofd`. + +```julia +vcat( + table(X, :Personal_Consumption, :Household_Supply, :Value_Added; normalize=:Use), + balance_of_payments(X, minimal=false), + table(X, :Investment_Final_Demand, :Government_Final_Demand; normalize=:Use) |> + x-> groupby(x, [:row, :year]) |> + x -> combine(x, + [:col, :parameter] .=> (y -> (:ofd)) .=> [:col, :parameter], + :value => sum => :value) +) +``` +""" +function representative_agent_data(X::T; output = :DataFrame) where T<:AbstractNationalTable + parameter_col(parameter) = parameter∈[:labor_demand, :capital_demand] ? :value_added : parameter + + out = vcat( table(X, :Personal_Consumption, :Household_Supply, :Value_Added; normalize=:Use), balance_of_payments(X, minimal=false), table(X, :Investment_Final_Demand, :Government_Final_Demand; normalize=:Use) |> @@ -206,16 +302,13 @@ function national_mpsge(data::T; year = 2023) where T<:AbstractNationalTable x -> combine(x, [:col, :parameter] .=> (y -> (:ofd)) .=> [:col, :parameter], :value => sum => :value) - ) |> - x -> DefaultDict(0, Dict((row[:row], row[:col], parameter_col(row[:parameter])) => row[:value] for row in eachrow(x))) |> - x -> @demand(M, RA, begin - @final_demand(PA[c=commodities], sum(x[c, pce, :personal_consumption] for pce in PCE)) # Negative ##NAICS - @endowment(PY[c=commodities], sum(x[c, hhs, :household_supply] for hhs in HHS)) ## NAICS - @endowment(PFX, x[:bop, :bop, :balance_of_payments]) - @endowment(PA[c=commodities], -x[c, :ofd, :ofd]) # Double negative - @endowment(PVA[va=value_added], sum(x[va, s, :value_added] for s∈sectors)) - end) - + ) - return M + if output == :DataFrame + return out + elseif output == :Dict + return DefaultDict(0, Dict((row[:row], row[:col], parameter_col(row[:parameter])) => row[:value] for row in eachrow(out))) + else + error("Unsupported output type: $output") + end end \ No newline at end of file diff --git a/src/united_states/build.jl b/src/united_states/build.jl index acc08f9..4d245bd 100644 --- a/src/united_states/build.jl +++ b/src/united_states/build.jl @@ -4,6 +4,15 @@ build_us_table(aggregation::Symbol = :summary) Build the US national table. Structure YAML file as specified by [`build_national_table`](@ref). + +The data is loaded and then transformed using the following processing steps: + +1. [`redistribute_cif_fob`])(@ref) +2. [`create_margin_categories`](@ref) +3. [`create_pce_categories`](@ref) +4. [`adjust_intermediate_flows`](@ref) +5. [`adjust_negative_value_added`](@ref) +6. [`zero_marginal_tax_subsidy`](@ref) """ function build_us_table(yaml_path::String; base_dir = pwd()) X, metadata = build_national_table(yaml_path, National; base_dir = base_dir) @@ -15,6 +24,8 @@ function build_us_table(yaml_path::String; base_dir = pwd()) X = create_margin_categories(X; transformation_keywords...) X = create_pce_categories(X; transformation_keywords...) X = adjust_intermediate_flows(X; transformation_keywords...) + X = adjust_negative_value_added(X; transformation_keywords...) + X = zero_marginal_tax_subsidy(X; transformation_keywords...) return X end @@ -240,4 +251,80 @@ function adjust_intermediate_flows(X::National; kwargs...) end +""" + adjust_negative_value_added(X::National; kwargs...) + +There are negative capital demeands in the US table. This function adjusts value added +to ensure that all capital demands are non-negative. The adjustment is given by + +```math +\\sum_{va} VA(year, va, sector) \\cdot \\frac{\\sum_{year} VA(year, va, sector)}{\\sum_{year, va} VA(year, va, sector)} +``` + +for any negative value in value added. + +Returns a National table. +""" +function adjust_negative_value_added(X::National; kwargs...) + DATA = table(X) + SETS = sets(X) + ELEMENTS = elements(X) + + shares = table(X, :Value_Added) |> + x -> groupby(x, [:col]) |> + x -> combine(x, + [:year, :row, :parameter, :value] .=> identity .=> [:year, :row, :parameter, :value], + :value => sum => :total_value + ) |> + x -> groupby(x, [:row, :col, :parameter]) |> + x -> combine(x, + :year => identity => :year, + [:value, :total_value] => ((v,tv) -> sum(v)./tv) => :share + ) + new_value_added = outerjoin( + table(X, :Value_Added), + shares, + on = [:year, :row, :col, :parameter] + ) |> + x -> groupby(x, [:col, :year]) |> + x -> combine(x, + [:row, :parameter] .=> identity .=> [:row, :parameter], + [:value, :share] => ( + (v,s) -> ifelse.(v.<0, v, s.*sum(v)) + ) => :value + ) + + va_params = elements(X, :Value_Added; base=true) |> x -> x[!,:set] + + DATA = DATA |> + x -> subset(x, + :parameter => ByRow(!∈(va_params)) + ) |> + x -> vcat(x, new_value_added) + + + + return National(DATA, SETS, ELEMENTS; regularity_check=true) +end + + +""" + zero_marginal_tax_subsidy(X::National; marginal_commodities::Vector{String} = [], kwargs...) + +Set the subsidy and tax parameters for marginal commodities to zero. + +Returns a National table. +""" +function zero_marginal_tax_subsidy(X::National; marginal_commodities::Vector{String} = [], kwargs...) + DATA = table(X) + SETS = sets(X) + ELEMENTS = elements(X) + + DATA = DATA |> + x -> subset(x, + [:row, :parameter] => ByRow((r,p) -> !( (r in Symbol.(marginal_commodities)) && (p in (:subsidy, :tax)) )) + ) + + return National(DATA, SETS, ELEMENTS; regularity_check=true) +end \ No newline at end of file diff --git a/src/united_states/detailed.yaml b/src/united_states/detailed.yaml index c528d9c..8f02c0d 100644 --- a/src/united_states/detailed.yaml +++ b/src/united_states/detailed.yaml @@ -3,17 +3,21 @@ metadata: "2017": 2017 "2012": 2012 "2007": 2007 - download_url_common: "https://apps.bea.gov/industry/iTables%20Static%20Files/AllTablesSUP.zip" + download_url_common: "https://apps.bea.gov/industry/release/zip/SUPPLY-USE.zip" download: supply: - pattern: "^Supply.*DET.xlsx$" + pattern: "^Supply_Detail.xlsx$" use: - pattern: "^Use.*_DET.xlsx$" + pattern: "^Use_SUT_Detail.xlsx$" transformation_keywords: insurance_codes: - "524113" - "5241XX" - "524200" + marginal_commodities: + - "441000" + - "445000" + - "452000" sets: commodity: description: "Commodities" diff --git a/src/united_states/summary.yaml b/src/united_states/summary.yaml index 8fa8ee2..f4afcae 100644 --- a/src/united_states/summary.yaml +++ b/src/united_states/summary.yaml @@ -27,15 +27,20 @@ metadata: "2021": 2021 "2022": 2022 "2023": 2023 - download_url_common: "https://apps.bea.gov/industry/iTables%20Static%20Files/AllTablesSUP.zip" + "2024": 2024 + download_url_common: "https://apps.bea.gov/industry/release/zip/SUPPLY-USE.zip" download: supply: - pattern: "^Supply.*Summary.xlsx$" + pattern: "^Supply_Summary.xlsx$" use: - pattern: "^Use.*_Summary.xlsx$" + pattern: "^Use_Summary.xlsx$" transformation_keywords: insurance_codes: - "524" + marginal_commodities: + - "441" + - "445" + - "452" na_values: - "..." sets: