Skip to content

Commit 0293bd1

Browse files
pascalwhooppascalwhoop
committed
adding introduction
1 parent b641f4b commit 0293bd1

File tree

3 files changed

+230
-51
lines changed

3 files changed

+230
-51
lines changed

‎src/bibliography.bib‎

Lines changed: 105 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,111 @@
1+
@misc{roberts_2016,
2+
title = {Why the "duck curve" created by solar power is a problem for utilities},
3+
url = {https://www.vox.com/2016/2/10/10960848/solar-energy-duck-curve},
4+
journal = {Vox},
5+
publisher = {Vox},
6+
author = {Roberts,
7+
David},
8+
year = {2016},
9+
month = {Feb}
10+
}
11+
12+
@misc{crawford_2015,
13+
title = {The California `Duck Curve' That Will Jolt Its Power Grid},
14+
url = {https://www.bloomberg.com/news/articles/2015-10-21/california-s-duck-curve-is-about-to-jolt-the-electricity-grid},
15+
journal = {Bloomberg.com},
16+
publisher = {Bloomberg},
17+
author = {Crawford, Jonathan},
18+
year = {2015},
19+
month = {Oct}}
20+
21+
@inproceedings{Molderink:2009:SEE:1995456.1995665,
22+
23+
author = {Molderink, Albert and Bosman, Maurice G. C. and Bakker, Vincent and Hurink, Johann L. and Smit, Gerard J.
24+
M.},
25+
title = {Simulating the Effect on the Energy Efficiency of Smart Grid Technologies},
26+
booktitle = {Winter Simulation Conference},
27+
series = {WSC '09},
28+
year = {2009},
29+
isbn = {978-1-4244-5771-7},
30+
location = {Austin, Texas},
31+
pages = {1530--1541},
32+
numpages = {12},
33+
url = {http://dl.acm.org/citation.cfm?id=1995456.1995665},
34+
acmid = {1995665},
35+
publisher = {Winter Simulation Conference},
36+
}
37+
38+
@article{Orgerie:2014:STI:2597757.2532637,
39+
author = {Orgerie, Anne-Cecile and Assuncao, Marcos Dias de and Lefevre, Laurent},
40+
title = {A Survey on Techniques for Improving the Energy Efficiency of Large-scale Distributed Systems},
41+
journal = {ACM Comput. Surv.},
42+
issue_date = {April 2014},
43+
volume = {46},
44+
number = {4},
45+
month = mar,
46+
year = {2014},
47+
issn = {0360-0300},
48+
pages = {47:1--47:31},
49+
articleno = {47},
50+
numpages = {31},
51+
url = {http://doi.acm.org/10.1145/2532637},
52+
doi = {10.1145/2532637},
53+
acmid = {2532637},
54+
publisher = {ACM},
55+
address = {New York, NY, USA},
56+
keywords = {Energy efficiency, computing, distributed systems, networking},
57+
}
58+
59+
@article{DePaola:2014:IMS:2620784.2611779,
60+
author = {De Paola, Alessandra and Ortolani, Marco and Lo Re, Giuseppe and Anastasi, Giuseppe and Das, Sajal K.},
61+
title = {Intelligent Management Systems for Energy Efficiency in Buildings: A Survey},
62+
journal = {ACM Comput. Surv.},
63+
issue_date = {July 2014},
64+
volume = {47},
65+
number = {1},
66+
month = jun,
67+
year = {2014},
68+
issn = {0360-0300},
69+
pages = {13:1--13:38},
70+
articleno = {13},
71+
numpages = {38},
72+
url = {http://doi.acm.org/10.1145/2611779},
73+
doi = {10.1145/2611779},
74+
acmid = {2611779},
75+
publisher = {ACM},
76+
address = {New York, NY, USA},
77+
keywords = {Building management systems, ambient intelligence, energy saving},
78+
}
79+
80+
@article{ketter2015competitive,
81+
title = {Competitive benchmarking: an IS research approach to address wicked problems with big data and analytics},
82+
author = {Ketter, Wolfgang and Peters, Markus and Collins, John and Gupta, Alok},
83+
year = {2015}
84+
}
85+
86+
@misc{mozur_markoff_2017,
87+
title = {Is China Outsmarting America in A.I.?},
88+
url = {https://www.nytimes.com/2017/05/27/technology/china-us-ai-artificial-intelligence.html},
89+
journal = {The New York Times},
90+
publisher = {The New York Times},
91+
author = {Mozur, Paul and Markoff, John},
92+
year = {2017},
93+
month = {May}
94+
}
95+
96+
@misc{faznetchina_2018,
97+
title = {Merkel in China: Die Kanzlerin kündigt Großes an},
98+
url = {http://www.faz.net/aktuell/wirtschaft/merkel-die-kooperation-mit-china-muss-jetzt-auf-ganz-neue-fuesse-gestellt-werden-15607145.html},
99+
journal = {FAZ.NET},
100+
publisher = {Frankfurter Allgemeine Zeitung},
101+
year = {2018},
102+
month = {May}
103+
}
104+
1105
@article{lillicrap2015continuous,
2106
title = {Continuous control with deep reinforcement learning},
3107
author = {Lillicrap, Timothy P and Hunt, Jonathan J and Pritzel, Alexander and Heess, Nicolas and Erez, Tom and
4-
Tassa, Yuval and Silver, David and Wierstra, Daan},
108+
Tassa, Yuval and Silver, David and Wierstra, Daan},
5109
journal = {arXiv preprint arXiv:1509.02971},
6110
year = {2015}
7111
}

‎src/body.tex‎

Lines changed: 100 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,85 @@
11
\chapter{Introduction}
22

3-
TODO Intro comes at the end
3+
Over the last few years, the field of \ac {AI} has seen a massive rise in publications and overall interest in the field
4+
\cite[]{arulkumaran2017brief, russell2016artificial}.
5+
It has been discussed as key future challenges for nation states and companies alike
6+
\cite[]{mozur_markoff_2017, faznetchina_2018}. Recent years have produced a large corpus of research focusing on visual data learning such
7+
as image recognition, audio and text based language recognition and robotics. In the field of \ac {RL}, many recent
8+
breakthroughs were achieved in robotics as well as common game challenges such as solving Atari games or playing Go
9+
\cite[]{arulkumaran2017brief}.
10+
11+
However, there are many other problem fields that can also benefit from such technologies. One such field is that of the
12+
global energy markets. These are expected to shift radically in the upcoming decades, adapting to new problems related to global warming and alternative
13+
energy sources. New problem solving techniques are required to solve such \emph{wicked problems}, because they depend on
14+
numerous impact factors such as economic, social, political and technical factors.
15+
\cite[]{ketter2015competitive}.
16+
17+
On a local scale, appliances need to improve their efficiency and machines need to deliver their performance with
18+
minimal energy requirements. Cars, fridges, water heating appliances, dishwashers and entertainment systems alike have
19+
all shown improvements in their efficiency and it has become a key component of a customers purchasing choice.
20+
Similarly, large distributed IT systems as well as building management systems are adapted to more efficiently make use
21+
of the energy they require
22+
\cite[]{Orgerie:2014:STI:2597757.2532637, DePaola:2014:IMS:2620784.2611779}.
23+
24+
On a regional and even national and international scale, the problem is equally complex. Energy systems were
25+
conventionally not built to contain \emph{energy buffers}. Energy always needed to be produced to match the demand. This
26+
is expected to change over the coming years due to an increasing number of electric vehicles and smart appliances. In
27+
addition, decentralized solar energy production changes the demand curve of macro-level energy supply. California is
28+
currently suffering a large supply of energy during sunny summer days while lacking energy when wind and solar energy
29+
output less due to lack of wind or sunshine. This puts previously unseen stress on the transport systems which were
30+
constructed to deliver large amounts of energy from few sources to many consumers instead of having many small producers
31+
distributed throughout the system
32+
\cite[]{roberts_2016}.
33+
34+
\ac {PowerTAC}, a competitive simulation of future energy markets, attempts to solve the planning dilemma of such
35+
complex systems. It allows researchers to experiment with numerous alternative scenarios, adapt the system dynamics to
36+
incentivize participants to behave in alignment with the greater interests and observe the interaction of a variety of
37+
market participants using different technologies to automatically generate profit. Researchers are invited to participate
38+
in this simulation by supplying usage models for appliances and developing \emph{brokers} that participate in the game.
39+
Brokers trade energy, offer contracts and coordinate storage capacities within their own customer network as well as
40+
with the overall market.
41+
42+
The simulation offers opportunities for several interesting fields of research: Game design, energy demand forecasting,
43+
intelligent contract design, commodity trading and of course general simulation and software design questions.
44+
45+
Brokers can be developed by anyone. This means that some broker developers have years of experience while others have
46+
not participated in a single competition. Each simulation takes approximately two to three hours to complete and each
47+
time-step takes five seconds. Previous researchers have identified the problem as a \ac {POMDP}, a common model of \ac
48+
{RL} literature \cite[]{tactexurieli2016mdp}. Deep \ac {NN} architectures have proven to be very successful in solving
49+
games in a variety of instances. It is therefore intuitive to attempt and apply such architectures to the problems posed
50+
by the \ac {PowerTAC} simulation. Unfortunately, most such implementations are only available in Python and \ac{PowerTAC}
51+
is almost exclusively based on Java. An extension of the current communication protocols to other languages may
52+
therefore benefit the overall reach of the simulation and motivate newcomers to join the competition with their Python
53+
based \ac {NN} architectures.
54+
55+
Finally, a subfield of \ac {RL} research has identified a problem in the transfer of knowledge from previously trained
56+
networks to newly developed iterations. Because \ac {NN} are mostly black boxes to researchers, it is difficult to
57+
extract knowledge and transfer this to another architecture. Especially when architectures differ in their
58+
hyperparameters, the learned weights of a \ac {NN} can not easily be transferred. The field of transfer learning has
59+
shown many interesting approaches for solving this problem. Agents with access to previously developed models may pass
60+
their observations to the \emph{teacher agent} and intially attempt to align their decisions to those that their teacher
61+
would do \cite[]{schmitt2018kickstarting}. More general problem solving agents may be trained by first training several
62+
small narrow focus agent networks on subproblems and then training the general agent on the actions of the narrow focus
63+
agents \cite[]{parisotto2015actor}. For problems where a reward function is difficult to construct, \emph{inverse
64+
reinforcement learning} can be used to train an agent to behave similar to an observable expert. The policy function of
65+
the agent shows good performance despite lacking a specific reward function \cite[]{NG2004Apprentice}.
66+
67+
To allow new brokers in the \ac {PowerTAC} setting to quickly catch up to previously developed competitor brokers,
68+
porting such learning transfer methods and their underlying deep architectures to the problem scope of \ac {PowerTAC}
69+
may be beneficial. The stated research question for this work therefore goes as follows:
70+
71+
\emph{Can \ac {RL} agents learn from actions of other agents in the \ac {PowerTAC} environment? If so, how? Can imitation allow for
72+
boosted performance of reinforcement learning algorithms within a competitive simulation environment?}
73+
74+
%TODO anything from the proposal that can be stolen?
475

576
% intro structuring basing on style from https://explorationsofstyle.com/2013/01/22/introductions/
677
%Intro short:
78+
% - recent developments of of A.I. and machine learnin
79+
% - most research problems applied to image recognition, translation and in the RL space to games and robotics.
780
% - global warming, lots of problems
881
% - reinvent the energy grid, lots of changes to the structure
982
% - very difficult to construct such a highly complex, globally spanning, must-never-fail system
10-
% - recent developments of of A.I. and machine learnin
1183
% - combine the two
1284

1385
%Intro long
@@ -25,30 +97,31 @@ \chapter{Introduction}
2597
%allow for boosted performance of reinforcement algorithms within a competitive simulation environment?}
2698
%-------------------------------------------------------------------------------
2799

28-
Global warming is a key challenge of the near and medium future. Without proper action, entire continents will see
29-
%TODO END
30-
31-
Global warming, if not combated, will change the face of the planet. Billions will be impacted, entire coastlines will
32-
be changed and cities all over the global will have to either be retrofitted to handle sub-sea level positioning or
33-
abandoned and relocated. (global warming report)
34-
35-
36-
One key component to avoid such disastrous effects is the reinvention of the energy systems of the world. While
37-
appliances on an individual level need to become ever more efficient, globally it is necessary to shift the
38-
transportation sector towards renewable energy sources.
39-
Solar and wind
40-
are required. But The future of energy is difficult (--> MISQ paper argumentation line)
41-
42-
Smart grids need decentralized intelligence where appliance level evaluation of the grid status impacts how energy is
43-
consumed. When such intelligence shifting is happening towards the \emph{edge} of the grid, it can be intelligent to
44-
introduce intermediate broker entities that mediate between the two extremes, the end-consumers and the wholesale
45-
market.
46-
47-
At the same time, current developments in AI and machine learning allow for highly sophisticated learning machines that
48-
can help manage complex tasks and systems. (citing some sexy AI papers)
49100

50-
Bringing these two developments together, it is intuitive to apply some of the recently developed technologies of
51-
\ac {AI} research to solve the coordination issues of contemporary, frankly crude energy networks.
101+
% Global warming is a key challenge of the near and medium future. Without proper action, entire continents will see
102+
% %TODO END
103+
%
104+
% Global warming, if not combated, will change the face of the planet. Billions will be impacted, entire coastlines will
105+
% be changed and cities all over the global will have to either be retrofitted to handle sub-sea level positioning or
106+
% abandoned and relocated. (global warming report)
107+
%
108+
%
109+
% One key component to avoid such disastrous effects is the reinvention of the energy systems of the world. While
110+
% appliances on an individual level need to become ever more efficient, globally it is necessary to shift the
111+
% transportation sector towards renewable energy sources.
112+
% Solar and wind
113+
% are required. But The future of energy is difficult (--> MISQ paper argumentation line)
114+
%
115+
% Smart grids need decentralized intelligence where appliance level evaluation of the grid status impacts how energy is
116+
% consumed. When such intelligence shifting is happening towards the \emph{edge} of the grid, it can be intelligent to
117+
% introduce intermediate broker entities that mediate between the two extremes, the end-consumers and the wholesale
118+
% market.
119+
%
120+
% At the same time, current developments in AI and machine learning allow for highly sophisticated learning machines that
121+
% can help manage complex tasks and systems. (citing some sexy AI papers)
122+
%
123+
% Bringing these two developments together, it is intuitive to apply some of the recently developed technologies of
124+
% \ac {AI} research to solve the coordination issues of contemporary, frankly crude energy networks.
52125

53126

54127
\section{Methodology}
@@ -641,6 +714,8 @@ \subsection{Deep Learning in Reinforcement Settings}%
641714

642715
\section{PowerTAC: A Competitive Simulation}
643716

717+
%TODO alternative sources / implementations like powertac
718+
% Simulating the effect on the energy efficiency of smart grid technologies.pdf
644719
In the following chapter, I will introduce the \acf{PowerTAC}. It's simulating a liberalized retail electrical energy
645720
market where multiple autonomous agents compete in different markets. Firstly, a retail market where agents, or
646721
\emph{brokers}, compete for numerous end-users through the offering of tariff contracts. Secondly, a wholesale market in

‎thesis.vim‎

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,27 @@
11
ab --- %-------------------------------------------------------------------------------
22
ab === %===============================================================================
3-
ab RL \ac {RL}
4-
ab GRPC \ac {GRPC}
5-
ab UL \ac {UL}
6-
ab kWh \ac {kWh}
7-
ab SL \ac {SL}
8-
ab RNN \ac {RNN}
9-
ab LSTM \ac {LSTM}
10-
ab DRL \ac {Deep RL}
11-
ab JMS \ac {JMS}
12-
ab XML \ac {XML}
13-
ab DU \ac {DU}
14-
ab CHP \ac {CHP}
15-
ab NN \ac {NN}
16-
ab SARSA \ac {SARSA}
17-
ab MDP \ac {MDP}
18-
ab POMDP \ac {POMDP}
19-
ab AI \ac {AI}
20-
ab PPO \ac {PPO}
21-
ab POMDP \ac {POMDP}
22-
ab GPU \ac {GPU}
23-
ab CPU \ac {CPU}
24-
ab TF \ac {TF}
25-
ab TPU \ac {TPU}
26-
ab SOTA \ac {SOTA}
27-
ab PowerTAC \ac {PowerTAC}
3+
ab RL \ac{RL}
4+
ab GRPC \ac{GRPC}
5+
ab UL \ac{UL}
6+
ab kWh \ac{kWh}
7+
ab SL \ac{SL}
8+
ab RNN \ac{RNN}
9+
ab LSTM \ac{LSTM}
10+
ab DRL \ac{Deep RL}
11+
ab JMS \ac{JMS}
12+
ab XML \ac{XML}
13+
ab DU \ac{DU}
14+
ab CHP \ac{CHP}
15+
ab NN \ac{NN}
16+
ab SARSA \ac{SARSA}
17+
ab MDP \ac{MDP}
18+
ab POMDP \ac{POMDP}
19+
ab AI \ac{AI}
20+
ab PPO \ac{PPO}
21+
ab POMDP \ac{POMDP}
22+
ab GPU \ac{GPU}
23+
ab CPU \ac{CPU}
24+
ab TF \ac{TF}
25+
ab TPU \ac{TPU}
26+
ab SOTA \ac{SOTA}
27+
ab PowerTAC \ac{PowerTAC}

0 commit comments

Comments
 (0)